diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py
index c6b732076..7f04be63d 100644
--- a/metagpt/memory/memory.py
+++ b/metagpt/memory/memory.py
@@ -26,10 +26,8 @@ class Memory:
         if message in self.storage:
             return
         self.storage.append(message)
-        # According to the design of RFC 116, it allows message filtering based on different labels, thus
-        # necessitating the creation of separate indices for each label.
-        for k in message.msg_to:
-            self.index[k].append(message)
+        if message.cause_by:
+            self.index[message.cause_by].append(message)
 
     def add_batch(self, messages: Iterable[Message]):
         for message in messages: