scripts/test_parser.py - main - mnskripnichenko/uvm-source-code-1.2-knowledge-mcp

scripts/test_parser.py
321 строка13 KB
luoyonghao1997
first commit
02 мар 2026, 10:18
02 мар 2026, 10:188bcd356
#!/usr/bin/env python3
"""
Parser self-test covering:
  1. Code block extraction (placeholder approach)
  2. HTML <img> tag alt-text extraction
  3. GitHub cross-file link resolution
  4. 被/调用 call-relation parsing
  5. class:method link-text parsing
  6. Markdown table conversion
  7. Anchor normalization (emoji, Chinese, mixed)
  8. Full card generation with call graph cards
"""

import re
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

from scripts.enrich_with_notes import (
    _build_section_index,
    _extract_call_relations,
    _extract_github_ref,
    _table_to_text,
    _to_anchor,
    build_relationship_cards,
    parse_markdown_file,
    resolve_cross_refs,
    section_to_card,
)

# ── Realistic sample files (mirroring actual note structure) ──────────────────

# Mirrors chapter 03: uvm_factory.md style
SAMPLE_FACTORY = """\
# 🗽 chapter 03: uvm_factory

| Parameter | Type | Usage |
|-----------|------|-------|
| m_types | associative array | Keeps track of all registered types |
| m_type_overrides | queue | Stores all type-based overrides |

### register
* 被 [uvm_component_registry:get](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2005%3A%20%20uvm_component_registry.md) 和 [uvm_object_registry:get](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2002%3A%20uvm_object_registry.md) 调用

<img width="940" alt="Factory Architecture" src="https://github.com/example-user/VLSI-Design-verification-interview/assets/66343787/d2f3f2d6">

```systemverilog
function void uvm_default_factory::register(uvm_object_wrapper obj);
  if (obj == null) begin
    uvm_report_fatal("NULLWR", "Attempting to register a null object", UVM_NONE);
  end
  m_types[obj] = 1;
endfunction
```

### set_type_override_by_type
* 被[uvm_component:set_type_override_by_type](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2006%3A%20uvm_component.md)调用
* 先检查传入的两个类型是否相同，如果相同直接退出。

### find_override_by_type
* 被 [create_object_by_type](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2003%3A%20uvm_factory.md#create_object_by_type) 调用
* 调用 [find_override_by_name](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2003%3A%20uvm_factory.md#find_override_by_name) 递归查找
"""

# Mirrors chapter 05: uvm_component_registry.md
SAMPLE_REGISTRY = """\
# 🗽 chapter 05: uvm_component_registry

### get
* 调用 [uvm_factory:register](https://github.com/example-user/uvm-notes/blob/main/03%3A%20uvm_base/chapter%2003%3A%20uvm_factory.md#register)

```systemverilog
static function this_type get();
  if (me == null) begin
    uvm_factory f = uvm_factory::get();
    me = new;
    f.register(me);
  end
  return me;
endfunction
```

* 在静态变量 me 初始化的时候被调用

![register flow diagram](https://github.com/example-user/uvm-notes/assets/image.png)
"""


def _write_temp(tmp_dir: Path, name: str, content: str) -> Path:
    p = tmp_dir / name
    p.write_text(content, encoding="utf-8")
    return p


# ── Tests ─────────────────────────────────────────────────────────────────────

def test_anchor_normalization() -> None:
    print("── test_anchor_normalization ──")
    cases = [
        ("🗽 chapter 03: uvm_factory", "chapter-03-uvm_factory"),
        ("uvm_phase_state_change", "uvm_phase_state_change"),
        ("工厂注册 Factory Registration", "工厂注册-factory-registration"),
        ("set_type_override_by_type", "set_type_override_by_type"),
        ("find_override_by_type", "find_override_by_type"),
        ("get", "get"),
    ]
    for heading, expected in cases:
        result = _to_anchor(heading)
        assert result == expected, f"FAIL: {heading!r} → {result!r} (expected {expected!r})"
        print(f"  ✓ {heading!r} → {result!r}")


def test_github_url_extraction() -> None:
    print("── test_github_url_extraction ──")
    cases = [
        (
            "https://github.com/example-user/VLSI-Design-verification-interview/blob/main/"
            "VLSI%20Design%20Verification/Functional%20Verification/"
            "UVM1.2%20source%20code%20analysis/03%3A%20uvm_base/"
            "chapter%2005%3A%20%20uvm_component_registry.md",
            "chapter 05:  uvm_component_registry.md", "",
        ),
        (
            "https://github.com/example-user/VLSI-Design-verification-interview/blob/main/"
            "VLSI%20Design%20Verification/Functional%20Verification/"
            "UVM1.2%20source%20code%20analysis/03%3A%20uvm_base/"
            "chapter%2003%3A%20uvm_factory.md#register",
            "chapter 03: uvm_factory.md", "register",
        ),
        ("https://github.com", "", ""),
        ("https://example.com/file.html#section", "", "section"),
    ]
    for url, expected_file, expected_anchor in cases:
        fname, anchor = _extract_github_ref(url)
        assert fname == expected_file, f"FAIL file: {url!r} → {fname!r}"
        assert anchor == expected_anchor, f"FAIL anchor: {url!r} → {anchor!r}"
        print(f"  ✓ → ({fname!r}, {anchor!r})")


def test_html_img_parsing(tmp_dir: Path) -> None:
    print("── test_html_img_parsing ──")
    fa = _write_temp(tmp_dir, "chapter 03: uvm_factory.md", SAMPLE_FACTORY)
    secs = parse_markdown_file(fa)

    # Find 'register' section
    reg_sec = next((s for s in secs if s.heading == "register"), None)
    assert reg_sec is not None, "register section not found"

    # Factory Architecture alt text should be collected
    all_alts = [a for s in secs for a in s.image_alts]
    print(f"  Image alts found: {all_alts}")
    assert any("Factory Architecture" in a or "architecture" in a.lower()
               for a in all_alts), \
        f"Expected 'Factory Architecture' in alts, got: {all_alts}"

    # No raw <img> or src= in plain text
    for sec in secs:
        assert "<img" not in sec.plain_text.lower(), \
            f"<img> leaked into plain text of '{sec.heading}'"
        assert "src=" not in sec.plain_text.lower(), \
            f"src= leaked into plain text of '{sec.heading}'"
    print("  ✓ HTML <img> parsed, alt text collected, no leakage into plain text")


def test_call_relation_parsing(tmp_dir: Path) -> None:
    print("── test_call_relation_parsing ──")
    fa = _write_temp(tmp_dir, "chapter 03: uvm_factory.md", SAMPLE_FACTORY)
    secs = parse_markdown_file(fa)

    # 'register' should be called_by uvm_component_registry:get AND uvm_object_registry:get
    reg_sec = next((s for s in secs if s.heading == "register"), None)
    assert reg_sec is not None
    print(f"  register call_relations: {[(r.direction, r.class_name, r.method_name) for r in reg_sec.call_relations]}")
    callers = [(r.class_name, r.method_name) for r in reg_sec.call_relations if r.direction == "called_by"]
    assert ("uvm_component_registry", "get") in callers, f"Missing caller, got: {callers}"
    assert ("uvm_object_registry", "get") in callers, f"Missing caller, got: {callers}"

    # 'set_type_override_by_type' should be called_by uvm_component:set_type_override_by_type
    ovrd_sec = next((s for s in secs if "set_type_override_by_type" in s.heading), None)
    assert ovrd_sec is not None
    callers2 = [(r.class_name, r.method_name) for r in ovrd_sec.call_relations if r.direction == "called_by"]
    print(f"  set_type_override_by_type callers: {callers2}")
    assert ("uvm_component", "set_type_override_by_type") in callers2

    print("  ✓ 被/调用 call relations correctly parsed")


def test_github_cross_ref_resolution(tmp_dir: Path) -> None:
    print("── test_github_cross_ref_resolution ──")
    fa = _write_temp(tmp_dir, "chapter 03: uvm_factory.md", SAMPLE_FACTORY)
    fb = _write_temp(tmp_dir, "chapter 05:  uvm_component_registry.md", SAMPLE_REGISTRY)

    secs_a = parse_markdown_file(fa)
    secs_b = parse_markdown_file(fb)
    all_secs = secs_a + secs_b

    # Show all cross-refs before resolution
    for sec in secs_a:
        for ref in sec.cross_refs:
            print(f"  cross-ref: [{ref.display_text}] → {ref.target_file}#{ref.target_anchor}")

    index = _build_section_index(all_secs)
    resolve_cross_refs(all_secs, index)

    resolved = [r for s in all_secs for r in s.cross_refs if r.target_heading]
    print(f"  Resolved cross-refs: {[(r.display_text, r.target_heading) for r in resolved]}")
    assert len(resolved) > 0, "Expected at least one resolved cross-ref"

    # 'register' section in factory should have incoming ref from registry:get
    reg_sec = next((s for s in secs_a if s.heading == "register"), None)
    assert reg_sec is not None
    print(f"  register incoming_refs: {reg_sec.incoming_refs}")
    assert len(reg_sec.incoming_refs) > 0, "register should have incoming refs"

    print("  ✓ GitHub cross-file references resolved correctly")


def test_table_conversion() -> None:
    print("── test_table_conversion ──")
    table = """\
| Parameter | Type | Usage |
|-----------|------|-------|
| m_types | associative array | Keeps track of all registered types |
| m_type_overrides | queue | Stores all type-based overrides |
"""
    result = _table_to_text(table)
    print(f"  Converted table:\n{result}")
    # Output format: "Parameter: m_types | Type: associative array | Usage: ..."
    assert "m_types" in result, "m_types value not found"
    assert "associative array" in result, "type value not found"
    assert "Keeps track" in result, "usage text not found"
    assert "m_type_overrides" in result, "m_type_overrides not found"
    assert "queue" in result, "queue type not found"
    # Table separator row (---) should NOT appear
    assert "---" not in result, "Table separator row leaked"
    print("  ✓ Table converted to readable key:value pairs")


def test_card_generation_with_call_graph(tmp_dir: Path) -> None:
    print("── test_card_generation_with_call_graph ──")
    fa = _write_temp(tmp_dir, "chapter 03: uvm_factory.md", SAMPLE_FACTORY)
    fb = _write_temp(tmp_dir, "chapter 05:  uvm_component_registry.md", SAMPLE_REGISTRY)

    secs = parse_markdown_file(fa) + parse_markdown_file(fb)
    index = _build_section_index(secs)
    resolve_cross_refs(secs, index)

    content_cards = [section_to_card(s) for s in secs if len(s.plain_text) >= 10 or s.code_snippets]
    rel_cards = build_relationship_cards(secs)
    call_cards = [c for c in rel_cards if c.get("card_type") == "call_graph"]

    print(f"  Content cards: {len(content_cards)}")
    print(f"  Relationship cards: {len([c for c in rel_cards if c.get('card_type')=='relationship'])}")
    print(f"  Call graph cards: {len(call_cards)}")

    assert len(call_cards) > 0, "Expected at least one call graph card"
    print(f"\n  Sample call-graph card:\n{call_cards[0]['text'][:300]}")

    # Check no raw markdown artefacts (images should be stripped; raw HTML should not appear)
    for card in content_cards:
        assert "![" not in card["text"], f"MD image leaked: {card['text'][:100]}"
        assert "<img" not in card["text"].lower(), f"HTML img leaked: {card['text'][:100]}"
        # Code blocks are now rendered as proper ``` fences — that is correct and expected.
        # What we check instead: the old [language] prefix artefact must be gone.
        assert not re.search(r"^\[(?:text|code|systemverilog)\]$", card["text"], re.MULTILINE), \
            f"Old [lang] prefix leaked: {card['text'][:150]}"

    # Check call metadata in content cards
    reg_card = next((c for c in content_cards if c.get("section_anchor") == "register"), None)
    assert reg_card is not None
    print(f"  register card called_by: {reg_card.get('called_by')}")
    assert len(reg_card.get("called_by", [])) >= 2, "register should have >= 2 callers"

    print("  ✓ All cards valid with call graph metadata")


def test_code_in_correct_section(tmp_dir: Path) -> None:
    print("── test_code_in_correct_section ──")
    fa = _write_temp(tmp_dir, "chapter 03: uvm_factory.md", SAMPLE_FACTORY)
    secs = parse_markdown_file(fa)

    reg_sec = next((s for s in secs if s.heading == "register"), None)
    assert reg_sec is not None
    assert len(reg_sec.code_snippets) >= 1, \
        f"register should have code snippet, got {len(reg_sec.code_snippets)}"
    assert "uvm_report_fatal" in reg_sec.code_snippets[0].content, \
        "Expected factory register code in snippet"
    print(f"  register section: {len(reg_sec.code_snippets)} code snippet(s)")
    print(f"  Snippet language: {reg_sec.code_snippets[0].language}")
    print(f"  Code preview: {reg_sec.code_snippets[0].content[:60]!r}...")
    print("  ✓ Code blocks attributed to correct section via placeholders")


def run_all_tests() -> None:
    import tempfile

    print("=" * 65)
    print("enrich_with_notes.py — Comprehensive Parser Self-Test")
    print("=" * 65)

    with tempfile.TemporaryDirectory() as tmp:
        tmp_dir = Path(tmp)

        test_anchor_normalization();      print()
        test_github_url_extraction();     print()
        test_html_img_parsing(tmp_dir);   print()
        test_call_relation_parsing(tmp_dir); print()
        test_github_cross_ref_resolution(tmp_dir); print()
        test_table_conversion();          print()
        test_code_in_correct_section(tmp_dir); print()
        test_card_generation_with_call_graph(tmp_dir)

    print()
    print("=" * 65)
    print("All tests passed ✓")
    print("=" * 65)


if __name__ == "__main__":
    run_all_tests()