From 795b892b3530d7dc97248593be72c7561dfabbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:24:54 +0800 Subject: [PATCH 1/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6d4e1b406..be262d606 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,6 +276,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @staticmethod def split_texts(text: str, window_size) -> List[str]: """Splitting long text into sliding windows text""" + if window_size <= 0: + window_size = OpenAIGPTAPI.DEFAULT_TOKEN_SIZE total_len = len(text) if total_len <= window_size: return [text] @@ -348,6 +350,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + DEFAULT_TOKEN_SIZE = 50 if __name__ == "__main__": From 67d08cb054cb863b1200a407b1d00bec42171c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:29:04 +0800 Subject: [PATCH 2/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index be262d606..dd5594b7d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if data_len + idx > total_len: windows.append(text[idx:]) break - w = text[idx:data_len] + w = text[idx : idx + data_len] windows.append(w) idx += data_len for i in range(len(windows)): From 614bdf9e742908be5e19a1fa938ec4fe135b2ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:43:58 +0800 Subject: [PATCH 3/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index dd5594b7d..64fbbdfd6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -286,13 +286,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows = [] idx = 0 while idx < total_len: - data_len = window_size - padding_size - if data_len + idx > total_len: + if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - w = text[idx : idx + data_len] + # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # window_size=3, padding_size=1： + # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... + # idx=2, | idx=5 | idx=8 | ... + w = text[idx : idx + window_size] windows.append(w) - idx += data_len + idx += window_size - padding_size if idx == 0 else window_size + for i in range(len(windows)): if i + 1 == len(windows): break From 0156fa592248d613ca2d4110fe563d0275eedd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:48:50 +0800 Subject: [PATCH 4/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 64fbbdfd6..019ad0b8b 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -285,6 +285,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if window_size > 20 else 0 windows = [] idx = 0 + data_len = window_size - padding_size while idx < total_len: if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) @@ -295,7 +296,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # idx=2, | idx=5 | idx=8 | ... w = text[idx : idx + window_size] windows.append(w) - idx += window_size - padding_size if idx == 0 else window_size + idx += data_len for i in range(len(windows)): if i + 1 == len(windows): From ea35305b52040c3da7e9efbe1b1c104f3f7c0603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:58:31 +0800 Subject: [PATCH 5/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 019ad0b8b..7ed9c0083 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] # window_size=3, padding_size=1： # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... # idx=2, | idx=5 | idx=8 | ... @@ -298,10 +298,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows.append(w) idx += data_len - for i in range(len(windows)): - if i + 1 == len(windows): - break - windows[i] += windows[i + 1][0:padding_size] return windows @staticmethod From 91595daa3b49f1a7bd0ed49e4bea80568455ba00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 23:14:07 +0800 Subject: [PATCH 6/6] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7ed9c0083..14347f20c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -351,7 +351,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_TOKEN_SIZE = 50 + DEFAULT_TOKEN_SIZE = 500 if __name__ == "__main__":