From 99774418aff62c7e3cbfc1bfc110a3014242beec Mon Sep 17 00:00:00 2001 From: seeker Date: Fri, 12 Apr 2024 11:38:15 +0800 Subject: [PATCH] update: The teminal tool adds Conda environment support for daemon mode running --- examples/di/run_flask.py | 20 ++++++++++ examples/di/use_github_repo.py | 3 +- metagpt/tools/libs/__init__.py | 2 + metagpt/tools/libs/terminal.py | 72 ++++++++++++++++++++++++++++------ 4 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 examples/di/run_flask.py diff --git a/examples/di/run_flask.py b/examples/di/run_flask.py new file mode 100644 index 000000000..ed0f35b8e --- /dev/null +++ b/examples/di/run_flask.py @@ -0,0 +1,20 @@ +import asyncio + +from metagpt.roles.di.data_interpreter import DataInterpreter + + +USE_GOT_REPO_REQ = """ +Write a service using Flask, create a conda environment and run it, and call the service's interface for validation. +Notice: Don't write all codes in one response, each time, just write code for one step. +""" +# If you have created a conda environment, you can say: +# I have created the conda environment '{env_name}', please use this environment to execute. + + +async def main(): + di = DataInterpreter(tools=["Terminal", "FileManager"]) + await di.run(USE_GOT_REPO_REQ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/di/use_github_repo.py b/examples/di/use_github_repo.py index ad541d2d9..7327f4597 100644 --- a/examples/di/use_github_repo.py +++ b/examples/di/use_github_repo.py @@ -5,7 +5,8 @@ from metagpt.roles.di.data_interpreter import DataInterpreter USE_GOT_REPO_REQ = """ This is a link to the GOT github repo: https://github.com/spcl/graph-of-thoughts.git. Clone it, read the README to understand the usage, install it, and finally run the quick start example. -**Note the config for LLM is at `config/config_got.json`, use this path directly.** Don't write all codes in one response, each time, just write code for one step. +**Note the config for LLM is at `config/config_got.json`, it's outside the repo path, before using it, you need to copy it into graph-of-thoughts. +** Don't write all codes in one response, each time, just write code for one step. """ diff --git a/metagpt/tools/libs/__init__.py b/metagpt/tools/libs/__init__.py index cd70d9811..fb96db735 100644 --- a/metagpt/tools/libs/__init__.py +++ b/metagpt/tools/libs/__init__.py @@ -12,6 +12,7 @@ from metagpt.tools.libs import ( web_scraping, email_login, terminal, + file_manager, ) from metagpt.tools.libs.software_development import ( write_prd, @@ -38,4 +39,5 @@ _ = ( fix_bug, git_archive, terminal, + file_manager, ) # Avoid pre-commit error diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index a23ebb86a..ee0d155cb 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -1,4 +1,6 @@ import subprocess +import threading +from queue import Queue from metagpt.logs import TOOL_LOG_END_MARKER, ToolLogItem, log_tool_output from metagpt.tools.tool_registry import register_tool @@ -6,7 +8,12 @@ from metagpt.tools.tool_registry import register_tool @register_tool() class Terminal: - """A tool for running terminal commands. Don't initialize a new instance of this class if one already exists.""" + """ + A tool for running terminal commands. + Don't initialize a new instance of this class if one already exists. + For commands that need to be executed within a Conda environment, it is recommended + to use the `execute_in_conda_env` method. + """ def __init__(self): self.shell_command = ["bash"] # FIXME: should consider windows support later @@ -21,20 +28,31 @@ class Terminal: text=True, bufsize=1, # Line buffered ) + self.stdout_queue = Queue() - def run_command(self, cmd: str) -> str: + def run_command(self, cmd: str, daemon=False) -> str: """ - Run a command in the terminal and return the output. - When the command is being executed, stream the output to the terminal. - Maintains state across commands, such as current directory. + Executes a specified command in the terminal and streams the output back in real time. + This command maintains state across executions, such as the current directory, + allowing for sequential commands to be contextually aware. The output from the + command execution is placed into `stdout_queue`, which can be consumed as needed. Args: - cmd (str): The command to run in the terminal. + cmd (str): The command to execute in the terminal. + daemon (bool): If True, executes the command in a background thread, allowing + the main program to continue execution. The command's output is + collected asynchronously in daemon mode and placed into `stdout_queue`. Returns: - str: The output of the terminal command. + str: The command's output or an empty string if `daemon` is True. Remember that + when `daemon` is True, the output is collected into `stdout_queue` and must + be consumed from there. + + Note: + If `stdout_queue` is not periodically consumed, it could potentially grow indefinitely, + consuming memory. Ensure that there's a mechanism in place to consume this queue, + especially during long-running or output-heavy command executions. """ - cmd_output = [] # Send the command self.process.stdin.write(cmd + self.command_terminator) @@ -42,6 +60,38 @@ class Terminal: f'echo "{TOOL_LOG_END_MARKER.value}"' + self.command_terminator ) # Unique marker to signal command end self.process.stdin.flush() + if daemon: + threading.Thread(target=self._read_and_process_output, args=(cmd,), daemon=True).start() + return "" + else: + return self._read_and_process_output(cmd) + + def execute_in_conda_env(self, cmd: str, env, daemon=False) -> str: + """ + Executes a given command within a specified Conda environment automatically without + the need for manual activation. Users just need to provide the name of the Conda + environment and the command to execute. + + Args: + cmd (str): The command to execute within the Conda environment. + env (str, optional): The name of the Conda environment to activate before executing the command. + If not specified, the command will run in the current active environment. + daemon (bool): If True, the command is run in a background thread, similar to `run_command`, + affecting error logging and handling in the same manner. + + Returns: + str: The command's output, or an empty string if `daemon` is True, with output processed + asynchronously in that case. + + Note: + This function wraps `run_command`, prepending the necessary Conda activation commands + to ensure the specified environment is active for the command's execution. + """ + cmd = f"conda run -n {env} {cmd}" + return self.run_command(cmd, daemon=daemon) + + def _read_and_process_output(self, cmd): + cmd_output = [] log_tool_output( output=ToolLogItem(name="cmd", value=cmd + self.command_terminator), tool_name="Terminal" ) # log the command @@ -52,10 +102,10 @@ class Terminal: if line.strip() == TOOL_LOG_END_MARKER.value: log_tool_output(TOOL_LOG_END_MARKER) break - log_tool_output( - output=ToolLogItem(name="output", value=line), tool_name="Terminal" - ) # log stdout in real-time + # log stdout in real-time + log_tool_output(output=ToolLogItem(name="output", value=line), tool_name="Terminal") cmd_output.append(line) + self.stdout_queue.put(line) return "".join(cmd_output)