trainable w/ chat formatting + mask only assistant response

hyperlora/data_utils.py

@@ -0,0 +1,264 @@
+import pandas as pd
+from typing import Literal
+from transformers import PreTrainedTokenizerBase
+
+from training_utils import TRAINING_TASK
+
+IGNORE_INDEX = -100
+
+
+def get_sft_prompt_formatting_fn(
+    sft_mode: TRAINING_TASK,
+    tokenizer: PreTrainedTokenizerBase,
+):
+    if sft_mode != TRAINING_TASK.COMPLETION:
+        raise NotImplementedError(
+            f"Training task {sft_mode} not supported. "
+            "Only completion is supported for now."
+        )
+
+    if tokenizer.chat_template is None:
+        raise NotImplementedError(
+            "Only chat models + SFT are supported. "
+            "Training with pre-training data is not supported yet."
+        )
+    # TODO: add support for causal_lm training (for pre-training data)
+    # TODO: add support for recon training (for pre-training data)
+    # TODO: add support for non-chat models
+
+    # def f(example):
+    #     output_texts = (
+    #         dict(text=[]) if sft_mode == "causal_lm" else dict(prompt=[], response=[])
+    #     )
+    #     df = pd.DataFrame(dict(example))
+    #     for i, inp_txt in df.iterrows():
+    #         if sft_mode == "causal_lm":
+    #             text = metadata["text_template"].format(**inp_txt)
+    #             output_texts["text"].append(text)
+    #         elif sft_mode == "completion":
+    #             prompt = metadata["user_prompt_template"].format(**inp_txt)
+    #             output_texts["prompt"].append(prompt)
+    #             output_texts["response"].append(str(inp_txt[metadata["response_field"]]))
+    #     return output_texts
+
+    def f_intx(example):
+        out = dict()
+        out["context"] = example["context"]
+
+        chat_text = tokenizer.apply_chat_template(
+            example["messages"], tokenize=False, add_generation_prompt=False
+        )
+        out["messages"] = example["messages"]
+        out["chat"] = chat_text
+        return out
+
+    # return f if not apply_chat_template_fn is not None else f_intx
+    return f_intx
+
+
+def convert_ctx_prompt_response_to_messages(example, add_ctx_to_chat=True):
+    if "prompt" not in example or "response" not in example:
+        raise ValueError(f"'prompt' and 'response' are required. Got: {example}")
+
+    system_msg = ""
+    if "system_message" in example:
+        system_msg = example["system_message"]
+
+    user_msg = example["prompt"]
+    if add_ctx_to_chat:
+        user_msg = example["context"] + "\n" + user_msg
+
+    example["messages"] = [
+        {"role": "system", "content": system_msg},
+        {"role": "user", "content": user_msg},
+        {"role": "assistant", "content": example["response"]},
+    ]
+
+    return example
+
+
+def get_preprocessing_fn(ds_name):
+    f = lambda x: x
+
+    # if ds_name == "context_numbers":
+
+    #     def f(example):
+    #         example["messages"] = [
+    #             {"role": "user", "content": example["prompt"]},
+    #             {"role": "assistant", "content": example["response"]},
+    #         ]
+    #         return example
+
+    # if ds_name.startswith("lol_"):
+
+    #     def f(example):
+    #         txt = example["input"]
+    #         task_def = txt.split("Definition: ")[1].split("\n\nPositive Example")[0]
+    #         task_def += " Please complete the task without any explanation."
+    #         if len(example["output"]) > 1:
+    #             task_def += "\nThe answer should be a comma-separated list of possible completions."
+    #         problem = txt.split("Now complete the following example -")[1].split("Input: ")[1].split("\nOutput:")[0]
+    #         answer = ", ".join(example["output"])
+    #         return dict(task_def=task_def, problem=problem, answer=answer)
+
+    # if ds_name.startswith("arc_"):
+    #     ABCD = ["A", "B", "C", "D"]
+
+    #     def f(example):
+    #         choices = example["choices"]
+    #         assert len(choices["text"]) == len(choices["label"])
+    #         n_to_fill = 4 - len(choices["text"])
+    #         if len(choices["text"]) < 4:
+    #             choices["text"] += ["N/A"] * n_to_fill
+    #         if len(choices["label"]) < 4:
+    #             if choices["label"][0].isdigit():
+    #                 choices["label"] += [str(len(choices["label"]) + i + 1) for i in range(n_to_fill)]
+    #             else:
+    #                 choices["label"] += [ABCD[len(choices["label"]) + i] for i in range(n_to_fill)]
+    #         example["choices"] = choices
+    #         return example
+
+    # if ds_name.startswith("mbpp"):
+    #     # for training an oracle lora on mbpp
+    #     def f(example):
+    #         example["assertions"] = "\n".join(example["test_list"])
+    #         return example
+
+    return f
+
+
+# def get_inp_tokenize_fn(
+#     tokenizer,
+#     sft_mode: Literal["causal_lm", "completion"],
+#     is_intx_model: bool,
+#     inp_max_len: int,
+# ):
+#     def tokenize_causal_lm(examples):
+#         # a dict with keys: ["input_ids", "attention_mask"]
+#         tokenized_seq = tokenizer(
+#             examples["text"],
+#             # apply_chat_template should already add all the special tokens
+#             add_special_tokens=True if not is_intx_model else False,
+#             truncation=True,
+#             padding=False,
+#             max_length=inp_max_len,
+#         )
+#         tokenized_seq["labels"] = tokenized_seq["input_ids"]
+#         return tokenized_seq
+
+#     # NOTE: we're not considering multi-turn sft
+#     # this fn is used to mask out the loss from the prompt
+#     # and train only on the response
+#     # see # see https://github.com/huggingface/trl/issues/632#issuecomment-1972630547
+#     # https://github.com/huggingface/notebooks/blob/main/examples/question_answering.ipynb
+#     # for more advanced multi-turn training
+#     def tokenize_prompt_completion(examples):
+#         # a dict with keys: ["input_ids", "attention_mask"]
+#         # we can also access seqeunce_ids to differentiate between prompt and response
+#         tokenized_seq = tokenizer(
+#             examples["prompt"],
+#             examples["response"],
+#             add_special_tokens=False,
+#             truncation=True,
+#             padding=False,
+#             max_length=inp_max_len,
+#         )
+
+#         tokenized_seq["labels"] = [None] * len(tokenized_seq["input_ids"])
+#         input_ids = tokenized_seq["input_ids"]
+#         attention_mask = tokenized_seq["attention_mask"]
+#         labels = tokenized_seq["labels"]
+#         for i in range(len(tokenized_seq["input_ids"])):
+#             if not is_intx_model:
+#                 # manually add bos and eos tokens
+#                 input_ids[i] = (
+#                     [tokenizer.bos_token_id] + input_ids[i] + [tokenizer.eos_token_id]
+#                 )
+#                 attention_mask[i] = [1] + attention_mask[i] + [1]
+#                 sequence_ids = [0] + tokenized_seq.sequence_ids(i) + [1]
+#             else:
+#                 sequence_ids = tokenized_seq.sequence_ids(i)
+#             labels[i] = [
+#                 -100 if sequence_id == 0 else label
+#                 for sequence_id, label in zip(sequence_ids, input_ids[i])
+#             ]
+#         return tokenized_seq
+
+#     tokenize_function = (
+#         tokenize_causal_lm if sft_mode == "causal_lm" else tokenize_prompt_completion
+#     )
+#     return tokenize_function
+
+
+def get_assistant_start_end_indices(messages, conversation_text):
+    """
+    Get the start and end indices of the assistant's messages in the conversation text.
+    """
+    start_indices = []
+    current_index = 0
+    for message in messages:
+        message_text = message["content"]
+        match_index = conversation_text[current_index:].find(message_text)
+        start_indices.append(current_index + match_index)
+        current_index += match_index + len(message_text)
+    end_indices = [
+        len(conversation_text) if i == len(start_indices) - 1 else start_indices[i + 1]
+        for i, x in enumerate(start_indices)
+    ]
+    roles = [message["role"] for message in messages]
+    return [
+        (s, e) for s, e, r in zip(start_indices, end_indices, roles) if r == "assistant"
+    ]
+
+
+def get_masked_labels(conversation_ids, assistant_ranges):
+    for id_, (id_s, id_e) in list(
+        zip(conversation_ids["input_ids"], conversation_ids["offset_mapping"])
+    ):
+        if any(id_s >= s and id_e <= e for s, e in assistant_ranges):
+            yield id_
+        else:
+            yield IGNORE_INDEX
+
+
+def tokenize_chat_messages(example, tokenizer, mask_assistant_inputs=True):
+    # should be used only with chat models
+    text = example["chat"]
+    messages = example["messages"]
+    conversation_ids = tokenizer(
+        text,
+        return_offsets_mapping=mask_assistant_inputs,
+        add_special_tokens=False,
+    )
+    if mask_assistant_inputs:
+        assistant_ranges = get_assistant_start_end_indices(messages, text)
+        labels = get_masked_labels(conversation_ids, assistant_ranges)
+        conversation_ids["labels"] = list(labels)
+        del conversation_ids["offset_mapping"]
+    else:
+        conversation_ids["labels"] = conversation_ids["input_ids"]
+    return conversation_ids
+
+
+if __name__ == "__main__":
+    from transformers import AutoTokenizer
+
+    model_name = "meta-llama/Llama-3.2-1B-Instruct"
+    messages = [
+        {"role": "user", "content": "Hello!"},
+        {"role": "assistant", "content": "Hey, how are you?"},
+        {"role": "user", "content": "Not too bad"},
+        {"role": "assistant", "content": "Cooooooool"},
+    ]
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    chat = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=False, add_special_tokens=False
+    )
+    print(chat)
+
+    model_inputs = tokenize_chat_messages(
+        {"chat": chat, "messages": messages}, tokenizer
+    )
+
+    print(tokenizer(chat))
+    breakpoint()

hyperlora/intx_sft.py

@@ -8,8 +8,15 @@ from transformers import (
     DataCollatorForSeq2Seq,
 )
 
+
 from modeling_utils import ModulatedPretrainedModel
-from training_utils import train_model
+from data_utils import (
+    convert_ctx_prompt_response_to_messages,
+    get_preprocessing_fn,
+    get_sft_prompt_formatting_fn,
+    tokenize_chat_messages,
+)
+from training_utils import TRAINING_TASK, train_model
 
 
 def compute_metrics(eval_pred) -> dict:
@@ -20,15 +27,6 @@ def compute_metrics(eval_pred) -> dict:
     Returns:
         dictionary containing metric names (str) and values (Any)
     """
-
-    # preds, labels = eval_preds
-
-    # # predictions is generated tokens for Seq2SeqTrainer
-    # # decode preds and labels
-    # labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-    # decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
-    # decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
-
     # compute per token accuracy
     preds, labels = eval_pred.predictions, eval_pred.label_ids
 
@@ -61,6 +59,7 @@ def main():
 
     model = AutoModelForCausalLM.from_pretrained(
         "meta-llama/Llama-3.1-8B-Instruct",
+        # "meta-llama/Llama-3.2-1B-Instruct",
         torch_dtype=torch.bfloat16,
         attn_implementation="flash_attention_2",
     )
@@ -68,59 +67,89 @@ def main():
     tokenizer.pad_token_id = tokenizer.eos_token_id
     tokenizer.padding_side = "right"
 
-    if isinstance(model, ModulatedPretrainedModel):
+    max_seq_len = 1024
 
-        def tokenize(example):
-            model_inputs = tokenizer(
-                example["prompt"],
-                truncation=True,
-                padding=False,
-            )
-            model_inputs["ctx_ids"] = tokenizer(example["context"]).input_ids
-            model_inputs["ctx_attention_mask"] = tokenizer(example["context"]).attention_mask
-            model_inputs["labels"] = ...
-            return model_inputs
+    # if isinstance(model, ModulatedPretrainedModel):
 
-    else:
+    #     # TODO: how to tokenize properly?
+    #     def tokenize(example):
+    #         model_inputs = tokenizer(
+    #             example["prompt"],
+    #             truncation=True,
+    #             padding=False,
+    #         )
+    #         model_inputs["ctx_ids"] = tokenizer(example["context"]).input_ids
+    #         model_inputs["ctx_attention_mask"] = tokenizer(
+    #             example["context"]
+    #         ).attention_mask
+    #         model_inputs["labels"] = ...
+    #         return model_inputs
 
-        def tokenize(example):
-            inp = [
-                ctx + "\n" + prompt for ctx, prompt in zip(example["context"], example["prompt"])
-            ]
-            model_inputs = tokenizer(
-                inp,
-                example["answer"],
-                # add_special_tokens=True, ???
-                truncation=True,
-                padding=False,
-            )
+    # else:
 
-            input_ids = model_inputs["input_ids"]
-            labels = [None] * len(input_ids)
+    #     def tokenize(example):
+    #         inp = [
+    #             ctx + "\n" + prompt
+    #             for ctx, prompt in zip(example["context"], example["prompt"])
+    #         ]
+    #         model_inputs = tokenizer(
+    #             inp,
+    #             example["answer"],
+    #             add_special_tokens=False,
+    #             truncation=True,
+    #             padding=False,
+    #             max_length=max_seq_len,
+    #         )
 
-            for i in range(len(input_ids)):
-                sequence_ids = model_inputs.sequence_ids(i)
-                labels[i] = [
-                    -100 if sequence_id == 0 else label
-                    for sequence_id, label in zip(sequence_ids, input_ids[i])
-                ]
-            model_inputs["labels"] = labels
-            return model_inputs
+    #         input_ids = model_inputs["input_ids"]
+    #         labels = [None] * len(input_ids)
+
+    #         for i in range(len(input_ids)):
+    #             sequence_ids = model_inputs.sequence_ids(i)
+    #             labels[i] = [
+    #                 -100 if sequence_id == 0 else label
+    #                 for sequence_id, label in zip(sequence_ids, input_ids[i])
+    #             ]
+    #         model_inputs["labels"] = labels
+    #         return model_inputs
 
     print("Loading dataset...")
     train_file = "../data/raw_datasets/context_numbers/train.jsonl"
     eval_file = "../data/raw_datasets/context_numbers/val.jsonl"
-    dataset = load_dataset("json", data_files={"train": train_file, "eval": eval_file})
-    train_ds = dataset["train"].map(tokenize, batched=True)
+    ds = load_dataset("json", data_files={"train": train_file, "eval": eval_file})
+    # preprocessing
+    ds = ds.map(get_preprocessing_fn("context_numbers"))
+    add_ctx_to_chat = not isinstance(model, ModulatedPretrainedModel)
+    # for sft + chat_model, we need to convert the dataset to chat format
+    # add "messages" field
+    ds = ds.map(
+        convert_ctx_prompt_response_to_messages,
+        fn_kwargs={"add_ctx_to_chat": add_ctx_to_chat},
+    )
+    # add "chat" field
+    ds = ds.map(get_sft_prompt_formatting_fn(TRAINING_TASK.COMPLETION, tokenizer))
+    # tokenize the chat + mask the assistant inputs
+    tokenized_ds = ds.map(
+        tokenize_chat_messages,
+        fn_kwargs={"tokenizer": tokenizer, "mask_assistant_inputs": True},
+        remove_columns=ds["train"].column_names,
+    )
+    train_ds = tokenized_ds["train"]
     eval_ds = {
-        "train": dataset["train"].select(range(100)).map(tokenize, batched=True),
-        "val": dataset["eval"].map(tokenize, batched=True),
+        "train": tokenized_ds["train"].select(range(100)),
+        "val": tokenized_ds["eval"],
     }
 
+    # train_ds = dataset["train"].map(tokenize, batched=True)
+    # eval_ds = {
+    #     "train": dataset["train"].select(range(100)).map(tokenize, batched=True),
+    #     "val": dataset["eval"].map(tokenize, batched=True),
+    # }
+
     # DataCollatorForSeq2Seq also pads the `labels`
     # useful when we're computing the labels manually
     # or masking the loss only on completion
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, pad_to_multiple_of=8)
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model, pad_to_multiple_of=8)
 
     train_model(
         model,

hyperlora/training_utils.py

@@ -1,6 +1,7 @@
 import math
 import os
 import random
+from enum import Enum, auto
 
 import torch
 from transformers import Seq2SeqTrainer, Trainer
@@ -9,6 +10,9 @@ from transformers.trainer_utils import get_last_checkpoint
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
+TRAINING_TASK = Enum("TRAINING_TASK", ["CAUSAL_LM", "COMPLETION"])
+
+
 def train_model(
     model,
     train_dataset,
@@ -19,7 +23,10 @@ def train_model(
 ):
 
     last_checkpoint = None
-    if os.path.isdir(training_args.output_dir) and not training_args.overwrite_output_dir:
+    if (
+        os.path.isdir(training_args.output_dir)
+        and not training_args.overwrite_output_dir
+    ):
         last_checkpoint = get_last_checkpoint(training_args.output_dir)
         if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
             raise ValueError(
@@ -27,7 +34,9 @@ def train_model(
                 " already exists and is not empty. "
                 "Use --overwrite_output_dir to overcome."
             )
-        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+        elif (
+            last_checkpoint is not None and training_args.resume_from_checkpoint is None
+        ):
             print(
                 f"Checkpoint detected, resuming training at {last_checkpoint}. "
                 "To avoid this behavior, change "
@@ -193,7 +202,9 @@ def instruct_ft_tokenize_function(examples, model, mem):
         total_mem_length = num_segments * model.mem_size
 
         prompt_ids = (
-            [mem[0]] * total_mem_length + [model.ft_token_id] + prompt_output["input_ids"][idx]
+            [mem[0]] * total_mem_length
+            + [model.ft_token_id]
+            + prompt_output["input_ids"][idx]
         )
         prompt_ids = (
             [1, 733, 16289, 28793] + prompt_ids + [733, 28748, 16289, 28793]
@@ -216,13 +227,20 @@ class DataCollatorForDynamicPadding:
         self.pad_to_multiple_of = pad_to_multiple_of
 
     def __call__(self, examples):
-        input_ids = [torch.tensor(example["input_ids"], dtype=torch.long) for example in examples]
-        labels = [torch.tensor(example["labels"], dtype=torch.long) for example in examples]
+        input_ids = [
+            torch.tensor(example["input_ids"], dtype=torch.long) for example in examples
+        ]
+        labels = [
+            torch.tensor(example["labels"], dtype=torch.long) for example in examples
+        ]
         prompt_answer_ids = [
-            torch.tensor(example["prompt_answer_ids"], dtype=torch.long) for example in examples
+            torch.tensor(example["prompt_answer_ids"], dtype=torch.long)
+            for example in examples
         ]
         input_ids = self.dynamic_padding(input_ids, fill_value=self.pad_token_id)
-        prompt_answer_ids = self.dynamic_padding(prompt_answer_ids, fill_value=self.pad_token_id)
+        prompt_answer_ids = self.dynamic_padding(
+            prompt_answer_ids, fill_value=self.pad_token_id
+        )
         labels = self.dynamic_padding(labels)
         batch = {
             "input_ids": input_ids,
@@ -237,7 +255,9 @@ class DataCollatorForDynamicPadding:
             max_length = (
                 (max_length - 1) // self.pad_to_multiple_of + 1
             ) * self.pad_to_multiple_of
-        padded_sequences = torch.full((len(sequences), max_length), fill_value, dtype=torch.long)
+        padded_sequences = torch.full(
+            (len(sequences), max_length), fill_value, dtype=torch.long
+        )
         for i, seq in enumerate(sequences):
             padded_sequences[i, : len(seq)] = seq
         return padded_sequences

icae_v2/dev_v2.jsonl

@@ -1,6 +1,6 @@
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "Identify the person arrested on suspicion of spying for North Korea.", "answer": "Benoit Quennedey"}
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "List the actions taken against Benoit Quennedey after his arrest.", "answer": "His office in the French Senate was raided by DGSI officers."}
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "What is the role of Benoit Quennedey in the French government?", "answer": "He is a senior civil servant who liaises between the French Senate and the Department of Architecture and Heritage."}
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "What are the charges against Benoit Quennedey?", "answer": "He is suspected of collecting and delivering to a foreign power information likely to subvert core national interests."}
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "Mention the organization Benoit Quennedey is believed to be the president of.", "answer": "Franco-Korean Friendship Association"}
-{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "When did the counterintelligence investigation into Quennedey's activities begin?", "answer": "In March of this year."}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "Identify the person arrested on suspicion of spying for North Korea.", "response": "Benoit Quennedey"}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "List the actions taken against Benoit Quennedey after his arrest.", "response": "His office in the French Senate was raided by DGSI officers."}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "What is the role of Benoit Quennedey in the French government?", "response": "He is a senior civil servant who liaises between the French Senate and the Department of Architecture and Heritage."}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "What are the charges against Benoit Quennedey?", "response": "He is suspected of collecting and delivering to a foreign power information likely to subvert core national interests."}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "Mention the organization Benoit Quennedey is believed to be the president of.", "response": "Franco-Korean Friendship Association"}
+{"context": "French senior civil servant arrested on suspicion of spying for North Korea\n\nNovember 27, 2018 by Joseph Fitsanakis\n\nA senior civil servant in the upper house of the French parliament has been arrested on suspicion of spying for North Korea, according to prosecutors. The news of the suspected spy\u2019s arrest was first reported on Monday by Quotidien, a daily politics and culture show on the Monaco-based television channel TMC. The show cited \u201ca judicial source in Paris\u201d and said that France\u2019s domestic security and counterintelligence agency, the General Directorate for Internal Security (DGSI), was in charge of the espionage case.\n\nThe senior administrator has been identified as Benoit Quennedey, a civil servant who liaises between the French Senate and the Department of Architecture and Heritage, which operates under France\u2019s Ministry of Culture. Quennedey was reportedly detained on Sunday morning and his office in the French Senate was raided by DGSI officers on the same day. Quotidien said that he was arrested on suspicion of \u201ccollecting and delivering to a foreign power information likely to subvert core national interests\u201d. The report did not provide specific information about the type of information that Quennedey is believed to have passed to North Korea. It did state, however, that a counterintelligence investigation into his activities began in March of this year.\n\nQuennedey is believed to be the president of the Franco-Korean Friendship Association, the French branch of a Spanish-based organization that lobbies in favor of international support for North Korea. Korea Friendship Association branches exist in over 30 countries and are believed to be officially sanctioned by Pyongyang. They operate as something akin to the pre-World War II Comintern (Communist International), a Moscow-sanctioned international pressure group that advocated in favor of Soviet-style communism around the world. French media reported on Monday that Quennedey traveled extensively to the Korean Peninsula in the past decade and has written a French-language book on North Korea. News reports said that the French President Emmanuel Macron had been made aware of Quennedey\u2019s arrest. The senior civil servant faces up to 30 years in prison if found guilty of espionage.\n\n\u25ba Author: Joseph Fitsanakis | Date: 27 November 2018 | Permalink\n\n", "prompt": "When did the counterintelligence investigation into Quennedey's activities begin?", "response": "In March of this year."}