apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-21 14:05:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1228 from YangQianli92/main

Browse source 
FIx:Correction of the calculation of the MRR indicator in the RAG benchmark
This commit is contained in:
Alexander Wu 2024-04-26 23:08:06 +08:00 • committed by GitHub
commit f201b2f5f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
metagpt/rag/benchmark/base.py
View file

@ -114,14 +114,14 @@ class RAGBenchmark:
def mean_reciprocal_rank(self, nodes: list[NodeWithScore], reference_docs: list[str]) -> float:
mrr_sum = 0.0
for i, doc in enumerate(reference_docs, start=1):
for node in nodes:
if node.text in doc:
for i, node in enumerate(nodes, start=1):
for doc in reference_docs:
if text in doc:
mrr_sum += 1.0 / i
break
return mrr_sum / len(reference_docs) if reference_docs else 0.0
return mrr_sum
return mrr_sum
async def semantic_similarity(self, response: str, reference: str) -> float:
result = await self.evaluator.aevaluate(
response=response,