From 68e87da378e5407d16c87370691083193e5c7042 Mon Sep 17 00:00:00 2001 From: Zhaoyang Yu Date: Tue, 10 Sep 2024 18:27:20 +0800 Subject: [PATCH 1/2] Update Hotpotqa --- examples/ags/benchmark/hotpotqa.py | 105 +++++------------- .../ags/experiments/baselines/cot_hotpotqa.py | 7 +- 2 files changed, 29 insertions(+), 83 deletions(-) diff --git a/examples/ags/benchmark/hotpotqa.py b/examples/ags/benchmark/hotpotqa.py index 375882511..1d977c11b 100644 --- a/examples/ags/benchmark/hotpotqa.py +++ b/examples/ags/benchmark/hotpotqa.py @@ -4,8 +4,12 @@ import aiofiles import pandas as pd import numpy as np from typing import List, Tuple, Callable, Set +from collections import Counter from tqdm.asyncio import tqdm_asyncio from scipy.optimize import linear_sum_assignment +import string +import re + from examples.ags.benchmark.utils import generate_random_indices @@ -16,9 +20,10 @@ def is_number(text: str) -> bool: except ValueError: return False -def normalize_answer(text): - import re - import string +def normalize_answer(s): + """ + Normalize answers for evaluation. + """ def remove_articles(text): return re.sub(r"\b(a|an|the)\b", " ", text) @@ -33,77 +38,24 @@ def normalize_answer(text): def lower(text): return text.lower() - def tokenize(text): - return re.split(" |-", text) + return white_space_fix(remove_articles(remove_punc(lower(s)))) - def normalize_number(text: str) -> str: - if is_number(text): - return str(float(text)) - else: - return text - - parts = [ - white_space_fix(remove_articles(normalize_number(remove_punc(lower(token))))) - for token in tokenize(text) - ] - parts = [part for part in parts if part.strip()] - normalized = " ".join(parts).strip() - return normalized - -def answer_to_bags(answer: str) -> Set[str]: - raw_spans = [answer] - - normalized_spans = [] - token_bags = [] - for raw_span in raw_spans: - normalized_span = normalize_answer(raw_span) - normalized_spans.append(normalized_span) - token_bags.append(set(normalized_span.split())) - return normalized_spans, token_bags - -def _align_bags(predicted: List[Set[str]], gold: List[Set[str]]) -> List[float]: +def f1_score(prediction, ground_truth): """ - Takes gold and predicted answer sets and first finds the optimal 1-1 alignment - between them and gets maximum metric values over all the answers. + Compute the F1 score between prediction and ground truth answers. """ - scores = np.zeros([len(gold), len(predicted)]) - for gold_index, gold_item in enumerate(gold): - for pred_index, pred_item in enumerate(predicted): - if match_numbers_if_present(gold_item, pred_item): - scores[gold_index, pred_index] = f1_score(pred_item, gold_item) - row_ind, col_ind = linear_sum_assignment(-scores) - - max_scores = np.zeros([max(len(gold), len(predicted))]) - for row, column in zip(row_ind, col_ind): - max_scores[row] = max(max_scores[row], scores[row, column]) - return max_scores - -def match_numbers_if_present(gold_bag: Set[str], predicted_bag: Set[str]) -> bool: - gold_numbers = set() - predicted_numbers = set() - for word in gold_bag: - if is_number(word): - gold_numbers.add(word) - for word in predicted_bag: - if is_number(word): - predicted_numbers.add(word) - if (not gold_numbers) or gold_numbers.intersection(predicted_numbers): - return True - return False - -def f1_score(predicted_bag: Set[str], gold_bag: Set[str]) -> float: - intersection = len(gold_bag.intersection(predicted_bag)) - if not predicted_bag: - precision = 1.0 - else: - precision = intersection / float(len(predicted_bag)) - if not gold_bag: - recall = 1.0 - else: - recall = intersection / float(len(gold_bag)) - f1 = (2 * precision * recall) / (precision + recall) if not (precision == 0.0 and recall == 0.0) else 0.0 + prediction_tokens = normalize_answer(prediction).split() + ground_truth_tokens = normalize_answer(ground_truth).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) return f1 + async def load_data(file_path: str, samples=20, total_length=1000) -> List[dict]: data = [] async with aiofiles.open(file_path, mode="r") as file: @@ -120,12 +72,8 @@ async def evaluate_problem(input: str, context_str: str, graph: Callable, expect while retries < max_retries: try: - prediction, supporting_sentences = await graph(input, context_str) if graph else "None" - predicted_bags = answer_to_bags(prediction) - gold_bags = answer_to_bags(expected_output) - - f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1]) - score = np.mean(f1_per_bag) + prediction = await graph(input, context_str) if graph else "None" + score = f1_score(prediction, expected_output) break except Exception as e: @@ -135,11 +83,10 @@ async def evaluate_problem(input: str, context_str: str, graph: Callable, expect if retries == max_retries: print("Maximum retries reached. Skipping this sample.") prediction = None - supporting_sentences = None score = 0 break - return input, prediction, expected_output, supporting_sentences, score + return input, prediction, expected_output, score async def evaluate_all_problems(data: List[dict], graph: Callable, max_concurrent_tasks: int = 50): semaphore = asyncio.Semaphore(max_concurrent_tasks) @@ -156,9 +103,9 @@ async def evaluate_all_problems(data: List[dict], graph: Callable, max_concurren return await tqdm_asyncio.gather(*tasks, desc="Evaluating HotpotQA problems", total=len(data)) -def save_results_to_csv(results: List[Tuple[str, str, str, str, float]], path: str) -> float: +def save_results_to_csv(results: List[Tuple[str, str, str, float]], path: str) -> float: df = pd.DataFrame( - results, columns=["question", "prediction", "expected_output", "supporting_sentences", "score"] + results, columns=["question", "prediction", "expected_output", "score"] ) average_score = df["score"].mean() diff --git a/examples/ags/experiments/baselines/cot_hotpotqa.py b/examples/ags/experiments/baselines/cot_hotpotqa.py index 22bd69438..b3a919fff 100644 --- a/examples/ags/experiments/baselines/cot_hotpotqa.py +++ b/examples/ags/experiments/baselines/cot_hotpotqa.py @@ -19,7 +19,6 @@ HOTPOTQA_PROMPT = """ class GenerateOp(BaseModel): answer: str = Field(default="", description="问题的答案") - supporting_sentences: str = Field(default="", description="支持性句子") class CoTGenerate(Operator): def __init__(self, llm: LLM, name: str = "Generate"): @@ -32,7 +31,7 @@ class CoTGenerate(Operator): fill_kwargs["mode"] = mode node = await ActionNode.from_pydantic(GenerateOp).fill(**fill_kwargs) response = node.instruct_content.model_dump() - return response["answer"], response["supporting_sentences"] + return response["answer"] class CoTSolveGraph(SolveGraph): def __init__(self, name: str, llm_config, dataset: str): @@ -40,8 +39,8 @@ class CoTSolveGraph(SolveGraph): self.cot_generate = CoTGenerate(self.llm) async def __call__(self, question: str, context: str) -> Tuple[str, str]: - answer, supporting_sentences = await self.cot_generate(question, context, mode="context_fill") - return answer, supporting_sentences + answer = await self.cot_generate(question, context, mode="context_fill") + return answer if __name__ == "__main__": async def main(): From 257b9944093f69459a76319ede220883403ab9da Mon Sep 17 00:00:00 2001 From: Zhaoyang Yu Date: Tue, 10 Sep 2024 18:45:54 +0800 Subject: [PATCH 2/2] Update drop.py --- examples/ags/benchmark/drop.py | 341 +++++++++++++++++++++++++-------- 1 file changed, 262 insertions(+), 79 deletions(-) diff --git a/examples/ags/benchmark/drop.py b/examples/ags/benchmark/drop.py index 1dbea0f50..7fc99134a 100644 --- a/examples/ags/benchmark/drop.py +++ b/examples/ags/benchmark/drop.py @@ -1,65 +1,166 @@ import json import asyncio import pandas as pd -from typing import List, Tuple, Callable, Dict, Any, Set +import string +import re +from typing import List, Tuple, Callable, Dict, Any, Set, Union import numpy as np from scipy.optimize import linear_sum_assignment from tqdm.asyncio import tqdm_asyncio from examples.ags.benchmark.utils import generate_random_indices -def is_number(text: str) -> bool: +def _remove_articles(text: str) -> str: + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + +def _white_space_fix(text: str) -> str: + return " ".join(text.split()) + + +EXCLUDE = set(string.punctuation) + +def _is_number(text: str) -> bool: try: float(text) return True except ValueError: return False -def normalize_answer(text): - import re - import string +def _normalize_number(text: str) -> str: + if _is_number(text): + return str(float(text)) + else: + return text - def remove_articles(text): - return re.sub(r"\b(a|an|the)\b", " ", text) +def _remove_punc(text: str) -> str: + if not _is_number(text): + return "".join(ch for ch in text if ch not in EXCLUDE) + else: + return text - def white_space_fix(text): - return " ".join(text.split()) - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) +def _lower(text: str) -> str: + return text.lower() - def lower(text): - return text.lower() - def tokenize(text): - return re.split(" |-", text) +def _tokenize(text: str) -> List[str]: + return re.split(" |-", text) - def normalize_number(text: str) -> str: - if is_number(text): - return str(float(text)) - else: - return text + +def _normalize_answer(text: str) -> str: + """Lower text and remove punctuation, articles and extra whitespace.""" parts = [ - white_space_fix(remove_articles(normalize_number(remove_punc(lower(token))))) - for token in tokenize(text) + _white_space_fix(_remove_articles(_normalize_number(_remove_punc(_lower(token))))) + for token in _tokenize(text) ] parts = [part for part in parts if part.strip()] normalized = " ".join(parts).strip() return normalized -def answer_to_bags(answer: str) -> Set[str]: - raw_spans = [answer] - - normalized_spans = [] +def _answer_to_bags( + answer: Union[str, List[str], Tuple[str, ...]] +) -> Tuple[List[str], List[Set[str]]]: + if isinstance(answer, (list, tuple)): + raw_spans = answer + else: + raw_spans = [answer] + normalized_spans: List[str] = [] token_bags = [] for raw_span in raw_spans: - normalized_span = normalize_answer(raw_span) + normalized_span = _normalize_answer(raw_span) normalized_spans.append(normalized_span) token_bags.append(set(normalized_span.split())) return normalized_spans, token_bags +#!/usr/bin/python + +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple, Union, Optional +import json +import argparse +import string +import re + +import numpy as np +from scipy.optimize import linear_sum_assignment + + +# From here through _normalize_answer was originally copied from: +# https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ +# Then cleaned up and modified a bit. +def _remove_articles(text: str) -> str: + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + +def _white_space_fix(text: str) -> str: + return " ".join(text.split()) + + +EXCLUDE = set(string.punctuation) + + +def _remove_punc(text: str) -> str: + if not _is_number(text): + return "".join(ch for ch in text if ch not in EXCLUDE) + else: + return text + + +def _lower(text: str) -> str: + return text.lower() + + +def _tokenize(text: str) -> List[str]: + return re.split(" |-", text) + + +def _normalize_answer(text: str) -> str: + """Lower text and remove punctuation, articles and extra whitespace.""" + + parts = [ + _white_space_fix(_remove_articles(_normalize_number(_remove_punc(_lower(token))))) + for token in _tokenize(text) + ] + parts = [part for part in parts if part.strip()] + normalized = " ".join(parts).strip() + return normalized + + +def _is_number(text: str) -> bool: + try: + float(text) + return True + except ValueError: + return False + + +def _normalize_number(text: str) -> str: + if _is_number(text): + return str(float(text)) + else: + return text + + +def _answer_to_bags( + answer: Union[str, List[str], Tuple[str, ...]] +) -> Tuple[List[str], List[Set[str]]]: + if isinstance(answer, (list, tuple)): + raw_spans = answer + else: + raw_spans = [answer] + normalized_spans: List[str] = [] + token_bags = [] + for raw_span in raw_spans: + normalized_span = _normalize_answer(raw_span) + normalized_spans.append(normalized_span) + token_bags.append(set(normalized_span.split())) + return normalized_spans, token_bags + + def _align_bags(predicted: List[Set[str]], gold: List[Set[str]]) -> List[float]: """ Takes gold and predicted answer sets and first finds the optimal 1-1 alignment @@ -68,8 +169,8 @@ def _align_bags(predicted: List[Set[str]], gold: List[Set[str]]) -> List[float]: scores = np.zeros([len(gold), len(predicted)]) for gold_index, gold_item in enumerate(gold): for pred_index, pred_item in enumerate(predicted): - if match_numbers_if_present(gold_item, pred_item): - scores[gold_index, pred_index] = f1_score(pred_item, gold_item) + if _match_numbers_if_present(gold_item, pred_item): + scores[gold_index, pred_index] = _compute_f1(pred_item, gold_item) row_ind, col_ind = linear_sum_assignment(-scores) max_scores = np.zeros([max(len(gold), len(predicted))]) @@ -77,20 +178,8 @@ def _align_bags(predicted: List[Set[str]], gold: List[Set[str]]) -> List[float]: max_scores[row] = max(max_scores[row], scores[row, column]) return max_scores -def match_numbers_if_present(gold_bag: Set[str], predicted_bag: Set[str]) -> bool: - gold_numbers = set() - predicted_numbers = set() - for word in gold_bag: - if is_number(word): - gold_numbers.add(word) - for word in predicted_bag: - if is_number(word): - predicted_numbers.add(word) - if (not gold_numbers) or gold_numbers.intersection(predicted_numbers): - return True - return False -def f1_score(predicted_bag: Set[str], gold_bag: Set[str]) -> float: +def _compute_f1(predicted_bag: Set[str], gold_bag: Set[str]) -> float: intersection = len(gold_bag.intersection(predicted_bag)) if not predicted_bag: precision = 1.0 @@ -100,9 +189,108 @@ def f1_score(predicted_bag: Set[str], gold_bag: Set[str]) -> float: recall = 1.0 else: recall = intersection / float(len(gold_bag)) - f1 = (2 * precision * recall) / (precision + recall) if not (precision == 0.0 and recall == 0.0) else 0.0 + f1 = ( + (2 * precision * recall) / (precision + recall) + if not (precision == 0.0 and recall == 0.0) + else 0.0 + ) return f1 +def _match_numbers_if_present(gold_bag: Set[str], predicted_bag: Set[str]) -> bool: + gold_numbers = set() + predicted_numbers = set() + for word in gold_bag: + if _is_number(word): + gold_numbers.add(word) + for word in predicted_bag: + if _is_number(word): + predicted_numbers.add(word) + if (not gold_numbers) or gold_numbers.intersection(predicted_numbers): + return True + return False + +def _compute_f1(predicted_bag: Set[str], gold_bag: Set[str]) -> float: + intersection = len(gold_bag.intersection(predicted_bag)) + if not predicted_bag: + precision = 1.0 + else: + precision = intersection / float(len(predicted_bag)) + if not gold_bag: + recall = 1.0 + else: + recall = intersection / float(len(gold_bag)) + f1 = ( + (2 * precision * recall) / (precision + recall) + if not (precision == 0.0 and recall == 0.0) + else 0.0 + ) + return f1 + +def _align_bags(predicted: List[Set[str]], gold: List[Set[str]]) -> List[float]: + """ + Takes gold and predicted answer sets and first finds the optimal 1-1 alignment + between them and gets maximum metric values over all the answers. + """ + scores = np.zeros([len(gold), len(predicted)]) + for gold_index, gold_item in enumerate(gold): + for pred_index, pred_item in enumerate(predicted): + if _match_numbers_if_present(gold_item, pred_item): + scores[gold_index, pred_index] = _compute_f1(pred_item, gold_item) + row_ind, col_ind = linear_sum_assignment(-scores) + + max_scores = np.zeros([max(len(gold), len(predicted))]) + for row, column in zip(row_ind, col_ind): + max_scores[row] = max(max_scores[row], scores[row, column]) + return max_scores + +def get_metrics( + predicted: Union[str, List[str], Tuple[str, ...]], gold: Union[str, List[str], Tuple[str, ...]] +) -> Tuple[float, float]: + """ + Takes a predicted answer and a gold answer (that are both either a string or a list of + strings), and returns exact match and the DROP F1 metric for the prediction. If you are + writing a script for evaluating objects in memory (say, the output of predictions during + validation, or while training), this is the function you want to call, after using + :func:`answer_json_to_strings` when reading the gold answer from the released data file. + """ + predicted_bags = _answer_to_bags(predicted) + gold_bags = _answer_to_bags(gold) + + if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]): + exact_match = 1.0 + else: + exact_match = 0.0 + + f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1]) + f1 = np.mean(f1_per_bag) + f1 = round(f1, 2) + return exact_match, f1 + +def answer_json_to_strings(answer: Dict[str, Any]) -> Tuple[Tuple[str, ...], str]: + """ + Takes an answer JSON blob from the DROP data release and converts it into strings used for + evaluation. + """ + if "number" in answer and answer["number"]: + return tuple([str(answer["number"])]), "number" + elif "spans" in answer and answer["spans"]: + return tuple(answer["spans"]), "span" if len(answer["spans"]) == 1 else "spans" + elif "date" in answer: + return ( + tuple( + [ + "{0} {1} {2}".format( + answer["date"]["day"], answer["date"]["month"], answer["date"]["year"] + ) + ] + ), + "date", + ) + else: + raise ValueError( + f"Answer type not found, should be one of number, spans or date at: {json.dumps(answer)}" + ) + def load_data(file_path: str, samples: int) -> List[Tuple[str, Dict[str, Any]]]: with open(file_path, mode="r") as file: data = json.load(file) @@ -113,44 +301,39 @@ def load_data(file_path: str, samples: int) -> List[Tuple[str, Dict[str, Any]]]: return data async def evaluate_problem(question: str, passage: str, answers: List[Dict[str, Any]], graph: Callable) -> Tuple[str, str, float]: - def answer_json_to_strings(answer: Dict[str, Any]) -> Tuple[Tuple[str, ...], str]: - if "number" in answer and answer["number"]: - return tuple([str(answer["number"])]), "number" - elif "spans" in answer and answer["spans"]: - return tuple(answer["spans"]), "span" if len(answer["spans"]) == 1 else "spans" - elif "date" in answer: - return ( - tuple( - [ - "{0} {1} {2}".format( - answer["date"]["day"], answer["date"]["month"], answer["date"]["year"] - ) - ] - ), - "date", - ) - else: - raise ValueError(f"Answer type not found, should be one of number, spans or date at: {json.dumps(answer)}") - prediction = await graph(question, passage) + max_retries = 5 + retries = 0 - def get_f1_score(prediction: str, golden_answer: str) -> float: - predicted_bags = answer_to_bags(prediction) - gold_bags = answer_to_bags(golden_answer) + while retries < max_retries: + try: + prediction = await graph(question, passage) - f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1]) - score = np.mean(f1_per_bag) - return score - max_score = 0.0 - best_answer = None - for answer in answers: - golden_answer, _ = answer_json_to_strings(answer) - golden_answer = golden_answer[0] - score = get_f1_score(prediction, golden_answer) - if score > max_score: - max_score = score - best_answer = golden_answer + max_score = 0.0 + max_type = None + best_answer = None + + for answer in answers: + golden_answer, golden_type = answer_json_to_strings(answer) + _, f1_score = get_metrics(prediction, golden_answer) + if golden_answer[0].strip() != "": + max_score = max(max_score, f1_score) + if max_score == f1_score: + max_type = golden_type + best_answer = golden_answer + break + + except Exception as e: + retries += 1 + print(f"Error generating prediction: {e}. Retrying... ({retries}/{max_retries})") + + if retries == max_retries: + print("Maximum retries reached. Skipping this sample.") + best_answer = None + prediction = None + max_score = 0.0 + break return best_answer, prediction, max_score @@ -165,7 +348,7 @@ async def evaluate_all_passages(annotations: List[Tuple[str, Dict[str, Any]]], g question = qa_pair["question"] answers = [qa_pair["answer"]] if "validated_answers" in qa_pair and qa_pair["validated_answers"]: - answers.extend(qa_pair["validated_answers"]) + answers += qa_pair["validated_answers"] best_answer, prediction, score = await evaluate_problem(question, passage, answers, graph) results.append([id, question, prediction, best_answer, score])