From f371e3a49979e87be1ce64b23b5d094b102cd271 Mon Sep 17 00:00:00 2001
From: geekan <geekan@foxmail.com>
Date: Tue, 19 Dec 2023 11:49:06 +0800
Subject: [PATCH] token_counter: add gpt-3.5-turbo-16k in list and add comment
 for them

---
 metagpt/utils/token_counter.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py
index 266a53268..ebfb85de7 100644
--- a/metagpt/utils/token_counter.py
+++ b/metagpt/utils/token_counter.py
@@ -56,6 +56,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
     if model in {
         "gpt-3.5-turbo-0613",
         "gpt-3.5-turbo-16k-0613",
+        "gpt-3.5-turbo-16k",
         "gpt-3.5-turbo-1106",
         "gpt-4-0314",
         "gpt-4-32k-0314",
@@ -63,7 +64,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
         "gpt-4-32k-0613",
         "gpt-4-1106-preview",
     }:
-        tokens_per_message = 3
+        tokens_per_message = 3  # # every reply is primed with <|start|>assistant<|message|>
         tokens_per_name = 1
     elif model == "gpt-3.5-turbo-0301":
         tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n