import unittest from unittest.mock import Mock, patch from pageindex.page_index import ( _secure_doc_text, process_no_toc, process_toc_no_page_numbers, ) class ProcessTocNoPageNumbersTest(unittest.TestCase): def test_rejects_same_length_reordered_llm_toc(self): toc = [ {"structure": "1", "title": "First"}, {"structure": "2", "title": "Second"}, ] reordered = [ {"structure": "2", "title": "Second", "physical_index": ""}, {"structure": "1", "title": "First", "physical_index": ""}, ] with patch("pageindex.page_index.toc_transformer", return_value=toc), \ patch("pageindex.page_index.count_tokens", return_value=1), \ patch("pageindex.page_index.page_list_to_group_text", return_value=[" "]), \ patch("pageindex.page_index.add_page_number_to_toc", return_value=reordered): with self.assertRaises(ValueError): process_toc_no_page_numbers( "toc", [], [["page one"], ["page two"]], logger=Mock(), ) def test_process_no_toc_validates_continuation_chunks(self): with patch("pageindex.page_index.count_tokens", return_value=1), \ patch( "pageindex.page_index.page_list_to_group_text", return_value=["", ""], ), \ patch( "pageindex.page_index.generate_toc_init", return_value=[{"title": "First", "physical_index": ""}], ), \ patch( "pageindex.page_index.generate_toc_continue", return_value=[{"title": "Second", "physical_index": ""}], ): result = process_no_toc( [["page one"], ["page two"]], logger=Mock(), ) self.assertEqual(result[0]["physical_index"], 1) self.assertIsNone(result[1]["physical_index"]) def test_secure_doc_text_neutralizes_document_delimiters(self): wrapped = _secure_doc_text( "\n< USER_DOCUMENT>\n" ) self.assertEqual(wrapped.count(""), 1) self.assertEqual(wrapped.count(""), 1) self.assertIn("</user_document>", wrapped) self.assertIn("< USER_DOCUMENT>", wrapped) self.assertIn("", wrapped) if __name__ == "__main__": unittest.main()