From e39cafdd580ebcf1b587b6de40a51b5deae3fef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 11 Mar 2024 22:25:38 +0800 Subject: [PATCH 1/6] feat: + tree command --- metagpt/utils/tree.py | 99 ++++++++++++++++++++++++++++++++ tests/metagpt/utils/test_tree.py | 54 +++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 metagpt/utils/tree.py create mode 100644 tests/metagpt/utils/test_tree.py diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py new file mode 100644 index 000000000..49b5634c6 --- /dev/null +++ b/metagpt/utils/tree.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/3/11 +@Author : mashenquan +@File : tree.py +@Desc : Implement the same functionality as the `tree` command. +Example: + root + +-- dir1 + | +-- file1.txt + | +-- file2.txt + +-- dir2 + | +-- subdir1 + | | +-- file1.txt + | | +-- file2.txt + | +-- subdir2 + | +-- file1.txt + | +-- file2.txt + +-- file.txt +""" +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Dict, List + +from anthropic import BaseModel +from pydantic import Field + + +class Tree(BaseModel): + """ + Represents a directory tree structure. + + Attributes: + root (str): The root directory of the tree. + tree (Dict[str, Dict]): The tree structure as a dictionary. + + Methods: + print: Print the directory tree structure. + + """ + + root: str + tree: Dict[str, Dict] = Field(default_factory=dict) + + def print(self, git_ignore_rules: Callable = None) -> str: + """ + Recursively traverses the directory structure and prints it out in a tree-like format. + + Args: + git_ignore_rules (Callable): Optional. A function to filter files to ignore. + + Returns: + str: A string representation of the directory tree. + + """ + root = Path(self.root).resolve() + self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) + v = self._print_tree(self.tree) + return "\n".join(v) + + @staticmethod + def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + tree = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + if i.is_file(): + tree[i.name] = {} + else: + tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) + return tree + + @staticmethod + def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: + ret = [] + for name, children in tree.items(): + ret.append(name) + if not children: + continue + lines = Tree._print_tree(tree=children, indent=indent + 1) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = Tree._add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + @staticmethod + def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py new file mode 100644 index 000000000..0d48f7ce3 --- /dev/null +++ b/tests/metagpt/utils/test_tree.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import List + +import pytest +from gitignore_parser import parse_gitignore + +from metagpt.utils.tree import Tree + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree(root: str, rules: str): + gitignore_rules = parse_gitignore(full_path=rules) if rules else None + tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) + assert tree + + +@pytest.mark.parametrize( + ("tree", "want"), + [ + ({"a": {"b": {}, "c": {}}}, ["a", "+-- b", "+-- c"]), + ({"a": {"b": {}, "c": {"d": {}}}}, ["a", "+-- b", "+-- c", " +-- d"]), + ( + {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}}, + ["a", "+-- b", "| +-- e", "| +-- f", "| +-- g", "+-- c", " +-- d"], + ), + ( + {"h": {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}, "i": {}}}, + [ + "h", + "+-- a", + "| +-- b", + "| | +-- e", + "| | +-- f", + "| | +-- g", + "| +-- c", + "| +-- d", + "+-- i", + ], + ), + ], +) +def test__print_tree(tree: dict, want: List[str]): + v = Tree._print_tree(tree) + assert v == want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 7d32f9efe6958bf9b76b4309d426c590e3268e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:10:09 +0800 Subject: [PATCH 2/6] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 218 ++++++++++++++++++++----------- tests/metagpt/utils/test_tree.py | 8 +- 2 files changed, 149 insertions(+), 77 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 49b5634c6..ad3373f5f 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -6,94 +6,166 @@ @File : tree.py @Desc : Implement the same functionality as the `tree` command. Example: - root - +-- dir1 - | +-- file1.txt - | +-- file2.txt - +-- dir2 - | +-- subdir1 - | | +-- file1.txt - | | +-- file2.txt - | +-- subdir2 - | +-- file1.txt - | +-- file2.txt - +-- file.txt + Usage: + >>> print_tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- dependency_file.py + +-- index.html + +-- make_sk_kernel.py + +-- token_counter.py + +-- embedding.py + +-- repair_llm_raw_output.py + +-- mermaid.py + +-- parse_html.py + +-- visual_graph_repo.py + +-- special_tokens.py + +-- ahttp_client.py + +-- __init__.py + +-- mmdc_ink.py + +-- di_graph_repository.py + +-- yaml_model.py + +-- cost_manager.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + | +-- mmdc_ink.cpython-39.pyc + | +-- read_document.cpython-39.pyc + | +-- mermaid.cpython-39.pyc + | +-- parse_html.cpython-39.pyc + | +-- human_interaction.cpython-39.pyc + | +-- cost_manager.cpython-39.pyc + | +-- json_to_markdown.cpython-39.pyc + | +-- graph_repository.cpython-39.pyc + | +-- ahttp_client.cpython-39.pyc + | +-- visual_graph_repo.cpython-39.pyc + | +-- file.cpython-39.pyc + | +-- di_graph_repository.cpython-39.pyc + | +-- pycst.cpython-39.pyc + | +-- save_code.cpython-39.pyc + | +-- dependency_file.cpython-39.pyc + | +-- text.cpython-39.pyc + | +-- token_counter.cpython-39.pyc + | +-- project_repo.cpython-39.pyc + | +-- yaml_model.cpython-39.pyc + | +-- serialize.cpython-39.pyc + | +-- git_repository.cpython-39.pyc + | +-- custom_decoder.cpython-39.pyc + | +-- parse_docstring.cpython-39.pyc + | +-- common.cpython-39.pyc + | +-- exceptions.cpython-39.pyc + | +-- repair_llm_raw_output.cpython-39.pyc + | +-- s3.cpython-39.pyc + | +-- embedding.cpython-39.pyc + | +-- make_sk_kernel.cpython-39.pyc + | +-- file_repository.cpython-39.pyc + +-- file.py + +-- save_code.py + +-- common.py + +-- redis.py + +-- text.py + +-- graph_repository.py + +-- singleton.py + +-- recovery_util.py + +-- file_repository.py + +-- pycst.py + +-- exceptions.py + +-- human_interaction.py + +-- highlight.py + +-- mmdc_pyppeteer.py + +-- s3.py + +-- json_to_markdown.py + +-- custom_decoder.py + +-- git_repository.py + +-- read_document.py + +-- parse_docstring.py """ from __future__ import annotations from pathlib import Path from typing import Callable, Dict, List -from anthropic import BaseModel -from pydantic import Field - -class Tree(BaseModel): +def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: """ - Represents a directory tree structure. + Recursively traverses the directory structure and prints it out in a tree-like format. - Attributes: - root (str): The root directory of the tree. - tree (Dict[str, Dict]): The tree structure as a dictionary. + Args: + root (str or Path): The root directory from which to start traversing. + git_ignore_rules (Callable): Optional. A function to filter files to ignore. - Methods: - print: Print the directory tree structure. + Returns: + str: A string representation of the directory tree. + + Example: + >>> tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + +-- parse_docstring.py + + >>> from gitignore_parser import parse_gitignore + >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- parse_docstring.py """ + root = Path(root).resolve() + dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} + v = _print_tree(dir_) + return "\n".join(v) - root: str - tree: Dict[str, Dict] = Field(default_factory=dict) - def print(self, git_ignore_rules: Callable = None) -> str: - """ - Recursively traverses the directory structure and prints it out in a tree-like format. - - Args: - git_ignore_rules (Callable): Optional. A function to filter files to ignore. - - Returns: - str: A string representation of the directory tree. - - """ - root = Path(self.root).resolve() - self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) - v = self._print_tree(self.tree) - return "\n".join(v) - - @staticmethod - def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: - tree = {} - for i in root.iterdir(): - if git_ignore_rules and git_ignore_rules(str(i)): - continue +def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + dir_ = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + try: if i.is_file(): - tree[i.name] = {} + dir_[i.name] = {} else: - tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) - return tree + dir_[i.name] = _list_children(root=i, git_ignore_rules=git_ignore_rules) + except (FileNotFoundError, PermissionError, OSError): + dir_[i.name] = {} + return dir_ - @staticmethod - def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: - ret = [] - for name, children in tree.items(): - ret.append(name) - if not children: - continue - lines = Tree._print_tree(tree=children, indent=indent + 1) - for j, v in enumerate(lines): - if v[0] not in ["+", " ", "|"]: - ret = Tree._add_line(ret) - row = f"+-- {v}" - else: - row = f" {v}" - ret.append(row) - return ret - @staticmethod - def _add_line(rows: List[str]) -> List[str]: - for i in range(len(rows) - 1, -1, -1): - v = rows[i] - if v[0] != " ": - return rows - rows[i] = "|" + v[1:] - return rows +def _print_tree(dir_: Dict[str:Dict]) -> List[str]: + ret = [] + for name, children in dir_.items(): + ret.append(name) + if not children: + continue + lines = _print_tree(children) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = _add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + +def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 0d48f7ce3..34eae10cf 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -4,7 +4,7 @@ from typing import List import pytest from gitignore_parser import parse_gitignore -from metagpt.utils.tree import Tree +from metagpt.utils.tree import _print_tree, tree @pytest.mark.parametrize( @@ -16,8 +16,8 @@ from metagpt.utils.tree import Tree ) def test_tree(root: str, rules: str): gitignore_rules = parse_gitignore(full_path=rules) if rules else None - tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) - assert tree + v = tree(root=root, git_ignore_rules=gitignore_rules) + assert v @pytest.mark.parametrize( @@ -46,7 +46,7 @@ def test_tree(root: str, rules: str): ], ) def test__print_tree(tree: dict, want: List[str]): - v = Tree._print_tree(tree) + v = _print_tree(tree) assert v == want From 6a8699cd4a8f0f2f6f3a8095c67bc72abe775b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:14:35 +0800 Subject: [PATCH 3/6] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 61 +------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index ad3373f5f..1c0060842 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -5,61 +5,18 @@ @Author : mashenquan @File : tree.py @Desc : Implement the same functionality as the `tree` command. -Example: - Usage: + Example: >>> print_tree(".") utils +-- serialize.py +-- project_repo.py +-- tree.py +-- mmdc_playwright.py - +-- dependency_file.py - +-- index.html - +-- make_sk_kernel.py - +-- token_counter.py - +-- embedding.py - +-- repair_llm_raw_output.py - +-- mermaid.py - +-- parse_html.py - +-- visual_graph_repo.py - +-- special_tokens.py - +-- ahttp_client.py - +-- __init__.py - +-- mmdc_ink.py - +-- di_graph_repository.py - +-- yaml_model.py +-- cost_manager.py +-- __pycache__ | +-- __init__.cpython-39.pyc | +-- redis.cpython-39.pyc | +-- singleton.cpython-39.pyc - | +-- mmdc_ink.cpython-39.pyc - | +-- read_document.cpython-39.pyc - | +-- mermaid.cpython-39.pyc - | +-- parse_html.cpython-39.pyc - | +-- human_interaction.cpython-39.pyc - | +-- cost_manager.cpython-39.pyc - | +-- json_to_markdown.cpython-39.pyc - | +-- graph_repository.cpython-39.pyc - | +-- ahttp_client.cpython-39.pyc - | +-- visual_graph_repo.cpython-39.pyc - | +-- file.cpython-39.pyc - | +-- di_graph_repository.cpython-39.pyc - | +-- pycst.cpython-39.pyc - | +-- save_code.cpython-39.pyc - | +-- dependency_file.cpython-39.pyc - | +-- text.cpython-39.pyc - | +-- token_counter.cpython-39.pyc - | +-- project_repo.cpython-39.pyc - | +-- yaml_model.cpython-39.pyc - | +-- serialize.cpython-39.pyc - | +-- git_repository.cpython-39.pyc - | +-- custom_decoder.cpython-39.pyc - | +-- parse_docstring.cpython-39.pyc - | +-- common.cpython-39.pyc - | +-- exceptions.cpython-39.pyc - | +-- repair_llm_raw_output.cpython-39.pyc - | +-- s3.cpython-39.pyc | +-- embedding.cpython-39.pyc | +-- make_sk_kernel.cpython-39.pyc | +-- file_repository.cpython-39.pyc @@ -67,22 +24,6 @@ Example: +-- save_code.py +-- common.py +-- redis.py - +-- text.py - +-- graph_repository.py - +-- singleton.py - +-- recovery_util.py - +-- file_repository.py - +-- pycst.py - +-- exceptions.py - +-- human_interaction.py - +-- highlight.py - +-- mmdc_pyppeteer.py - +-- s3.py - +-- json_to_markdown.py - +-- custom_decoder.py - +-- git_repository.py - +-- read_document.py - +-- parse_docstring.py """ from __future__ import annotations From 684730e94f7c5fe7f5bac8a31ed8fac1937b6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:06:40 +0800 Subject: [PATCH 4/6] feat: +`tree` command --- metagpt/utils/tree.py | 36 ++++++++++++++++++++++++++++---- tests/metagpt/utils/test_tree.py | 16 +++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 1c0060842..c0386d822 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -27,17 +27,22 @@ """ from __future__ import annotations +import subprocess from pathlib import Path from typing import Callable, Dict, List +from gitignore_parser import parse_gitignore -def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: + +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. Args: root (str or Path): The root directory from which to start traversing. - git_ignore_rules (Callable): Optional. A function to filter files to ignore. + gitignore (str or Path): The filename of gitignore file. + run_command (bool): Whether to execute `tree` command. Execute the `tree` command and return the result if True, + otherwise execute python code instead. Returns: str: A string representation of the directory tree. @@ -55,8 +60,7 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: | +-- singleton.cpython-39.pyc +-- parse_docstring.py - >>> from gitignore_parser import parse_gitignore - >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + >>> tree(".", gitignore="../../.gitignore") utils +-- serialize.py +-- project_repo.py @@ -64,8 +68,21 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: +-- mmdc_playwright.py +-- parse_docstring.py + >>> tree(".", gitignore="../../.gitignore", run_command=True) + utils + ├── serialize.py + ├── project_repo.py + ├── tree.py + ├── mmdc_playwright.py + └── parse_docstring.py + + """ root = Path(root).resolve() + if run_command: + return _execute_tree(root, gitignore) + + git_ignore_rules = parse_gitignore(gitignore) if gitignore else None dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} v = _print_tree(dir_) return "\n".join(v) @@ -110,3 +127,14 @@ def _add_line(rows: List[str]) -> List[str]: return rows rows[i] = "|" + v[1:] return rows + + +def _execute_tree(root: Path, gitignore: str | Path) -> str: + args = ["--gitignore", str(gitignore)] if gitignore else [] + try: + result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) + if result.returncode != 0: + raise ValueError(f"tree exits with code {result.returncode}") + return result.stdout + except subprocess.CalledProcessError as e: + raise e diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 34eae10cf..03a2a5606 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import List import pytest -from gitignore_parser import parse_gitignore from metagpt.utils.tree import _print_tree, tree @@ -15,8 +14,19 @@ from metagpt.utils.tree import _print_tree, tree ], ) def test_tree(root: str, rules: str): - gitignore_rules = parse_gitignore(full_path=rules) if rules else None - v = tree(root=root, git_ignore_rules=gitignore_rules) + v = tree(root=root, gitignore=rules) + assert v + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree_command(root: str, rules: str): + v = tree(root=root, gitignore=rules, run_command=True) assert v From 6f4d30825f84d90c35dab04e639fc1b483e1a823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:10:13 +0800 Subject: [PATCH 5/6] feat: +`tree` command --- metagpt/utils/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index c0386d822..fbf085e48 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -34,7 +34,7 @@ from typing import Callable, Dict, List from gitignore_parser import parse_gitignore -def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = False) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. From 9cfcfb1ea8bae086ed4fc5fe9d8365038c95e89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 15:14:14 +0800 Subject: [PATCH 6/6] feat: use --gitfile --- .gitignore | 2 +- metagpt/utils/tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6bc67fa61..1542bbb98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: