PageIndex/tests/test_markdown_parser.py

import pytest
from pathlib import Path
from pageindex.parser.markdown import MarkdownParser
from pageindex.parser.protocol import ContentNode, ParsedDocument

@pytest.fixture
def sample_md(tmp_path):
    md = tmp_path / "test.md"
    md.write_text("""# Chapter 1
Some intro text.

## Section 1.1
Details here.

## Section 1.2
More details.

# Chapter 2
Another chapter.
""")
    return str(md)

def test_supported_extensions():
    parser = MarkdownParser()
    exts = parser.supported_extensions()
    assert ".md" in exts
    assert ".markdown" in exts

def test_parse_returns_parsed_document(sample_md):
    parser = MarkdownParser()
    result = parser.parse(sample_md)
    assert isinstance(result, ParsedDocument)
    assert result.doc_name == "test"

def test_parse_nodes_have_level(sample_md):
    parser = MarkdownParser()
    result = parser.parse(sample_md)
    assert len(result.nodes) == 4
    assert result.nodes[0].level == 1
    assert result.nodes[0].title == "Chapter 1"
    assert result.nodes[1].level == 2
    assert result.nodes[1].title == "Section 1.1"
    assert result.nodes[3].level == 1

def test_parse_nodes_have_content(sample_md):
    parser = MarkdownParser()
    result = parser.parse(sample_md)
    assert "Some intro text" in result.nodes[0].content
    assert "Details here" in result.nodes[1].content

def test_parse_nodes_have_index(sample_md):
    parser = MarkdownParser()
    result = parser.parse(sample_md)
    for node in result.nodes:
        assert node.index is not None


def test_preamble_before_first_header_is_kept(tmp_path):
    md = tmp_path / "pre.md"
    md.write_text("Abstract: important preamble text.\n\n# Chapter 1\nBody.\n")
    result = MarkdownParser().parse(str(md))
    assert result.nodes[0].title == "pre"
    assert "important preamble text" in result.nodes[0].content
    assert result.nodes[1].title == "Chapter 1"


def test_headerless_file_yields_single_node(tmp_path):
    md = tmp_path / "plain.md"
    md.write_text("Just some text.\nNo headings at all.\n")
    result = MarkdownParser().parse(str(md))
    assert len(result.nodes) == 1
    assert result.nodes[0].title == "plain"
    assert "No headings at all" in result.nodes[0].content


def test_utf8_bom_does_not_break_the_first_header(tmp_path):
    """A leading BOM (common from Windows editors/exporters) isn't
    whitespace, so .strip() doesn't remove it — without utf-8-sig decoding,
    the header regex fails to match the BOM-prefixed first line, and it gets
    misclassified as unrecognized preamble text instead of a real heading."""
    md = tmp_path / "bom.md"
    md.write_bytes(b"\xef\xbb\xbf# First Header\nbody text\n")
    result = MarkdownParser().parse(str(md))
    assert len(result.nodes) == 1
    assert result.nodes[0].title == "First Header"
    assert result.nodes[0].level == 1


def test_tilde_fenced_code_blocks_are_recognized(tmp_path):
    """CommonMark allows both backtick and tilde code fences. Only
    recognizing backticks let a '#'-prefixed line inside a ~~~-fenced block
    (e.g. a shell comment in a sample) be misparsed as a real heading."""
    md = tmp_path / "tilde.md"
    md.write_text(
        "# Real Header\nintro\n"
        "~~~\n# not a real header, just a comment\n~~~\n"
        "## Real Sub\nmore\n"
    )
    result = MarkdownParser().parse(str(md))
    titles = [n.title for n in result.nodes]
    assert titles == ["Real Header", "Real Sub"]
    assert "not a real header" not in " ".join(n.title for n in result.nodes)