mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
icml cleaned
This commit is contained in:
51616
parent
c4c1ac6eaa
commit
8093817c34
30 changed files with 366 additions and 1094 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -6,6 +6,7 @@ models/
|
|||
results/
|
||||
datasets/
|
||||
llm-comparator/
|
||||
trained_d2l/
|
||||
# Ignore data files but not scripts
|
||||
/data/processed_datasets/
|
||||
/data/raw_datasets/
|
||||
|
|
|
|||
|
|
@ -18,11 +18,6 @@ max_packed_ctx_len: 6144
|
|||
# data
|
||||
train_ds_names:
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/train/*level_1*.parquet
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/fw_qa_v2/min_0_to_2000/train/*level_3*.parquet
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/squad_compact
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/ropes_compact
|
||||
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/drop_compact
|
||||
|
||||
val_ds_names:
|
||||
- squad
|
||||
|
|
@ -246,7 +246,7 @@ def main():
|
|||
k_values = list(range(k_start, k_end))
|
||||
bin_size = len(k_values)
|
||||
save_dir = f"{args.out_prefix}_{tok_bin[0]}_{tok_bin[1]}"
|
||||
training_enabled = tok_bin[1] <= 512 # unchanged policy
|
||||
training_enabled = tok_bin[1] <= 1024 # unchanged policy
|
||||
if training_enabled:
|
||||
train_data: list[dict] = []
|
||||
# Distribute training budget across k values.
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ except FileNotFoundError:
|
|||
DEFAULT_CONTEXT = ""
|
||||
|
||||
WARNING_MESSAGE = (
|
||||
"⚠️ **Caution**: This is an academic proof-of-concept demonstration.\n"
|
||||
"⚠️ **Caution**: This is an educational proof-of-concept demonstration.\n"
|
||||
"The model may generate inaccurate information or hallucinate facts."
|
||||
)
|
||||
|
||||
|
|
@ -665,6 +665,7 @@ def create_demo():
|
|||
|
||||
chatbot = gr.Chatbot(
|
||||
label="Conversation",
|
||||
show_copy_button=True,
|
||||
height=500,
|
||||
elem_id="chatbot",
|
||||
elem_classes="chat-container",
|
||||
|
|
@ -785,7 +786,7 @@ if __name__ == "__main__":
|
|||
demo = create_demo()
|
||||
demo.launch(
|
||||
server_name="0.0.0.0",
|
||||
server_port=7860,
|
||||
server_port=7861,
|
||||
share=False,
|
||||
debug=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -106,6 +106,12 @@ if __name__ == "__main__":
|
|||
default=4,
|
||||
help="Number of rounds of query generation for context distillation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cd_batch_size",
|
||||
type=int,
|
||||
default=16,
|
||||
help="Batch size for context distillation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use_iterative_mode",
|
||||
action="store_true",
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
# TO BE ADDED: see which split is better
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
if __name__ == "__main__":
|
||||
fw_dir = "./data/raw_datasets/self_gen/"
|
||||
self_gen_data_dir = "./data/raw_datasets/self_gen/"
|
||||
snapshot_download(
|
||||
"SakanaAI/self_gen_qa_fw_qa",
|
||||
"SakanaAI/SakanaAI/self_gen_qa_d2l",
|
||||
repo_type="dataset",
|
||||
local_dir=fw_dir,
|
||||
local_dir=self_gen_data_dir,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
# no truncation
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --eval_batch_size_gen 1 --max_test_samples_per_ds 10
|
||||
|
||||
# w/ truncation
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --eval_batch_size_gen 1 --truncate_if_too_long_inp --max_test_samples_per_ds 10
|
||||
|
||||
# no context
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --eval_batch_size_gen 1 --remove_context --max_test_samples_per_ds 10
|
||||
|
|
@ -3,4 +3,4 @@ WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it
|
|||
|
||||
|
||||
# longbench
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench/multifieldqa_en_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --cd_use_gen_q --q_gen_rounds=1
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench/multifieldqa_en_e longbench/2wikimqa_e longbench/qasper_e --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --cd_use_gen_q --q_gen_rounds=1
|
||||
|
|
|
|||
1
scripts/main_exp/eval/cd_minibatch.sh
Normal file
1
scripts/main_exp/eval/cd_minibatch.sh
Normal file
|
|
@ -0,0 +1 @@
|
|||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets $1 --split test --use_cd --cd_update_iterations 50 --eval_batch_size_gen=1 --truncate_if_too_long_inp --cd_use_gen_q --q_gen_rounds=5 --cd_batch_size=2
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# qa
|
||||
uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --max_test_samples_per_ds 10
|
||||
|
||||
|
||||
# longbench
|
||||
uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench/multifieldqa_en_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --max_test_samples_per_ds 10
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
# qa
|
||||
uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --cd_use_gen_q --q_gen_rounds=4 --max_test_samples_per_ds 10
|
||||
|
||||
|
||||
# longbench
|
||||
uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench/multifieldqa_en_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --use_cd --cd_update_iterations 300 --eval_batch_size_gen=1 --truncate_if_too_long_inp --cd_use_gen_q --q_gen_rounds=1 --max_test_samples_per_ds 10
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# main results
|
||||
# batched
|
||||
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path ../../ctx-to-lora/train_outputs/runs/Sep29_14-42-46_slurm0-a3nodeset-9_88483_1e7bb34e/checkpoint-40000/pytorch_model.bin --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --max_ctx_chunk_len 8192 --eval_batch_size_gen 1 --max_test_samples_per_ds 10
|
||||
|
||||
# iterative
|
||||
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path ../../ctx-to-lora/train_outputs/runs/Sep29_14-42-46_slurm0-a3nodeset-9_88483_1e7bb34e/checkpoint-40000/pytorch_model.bin --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --max_ctx_chunk_len 8192 --eval_batch_size_gen 1 --use_iterative_mode --max_test_samples_per_ds 10
|
||||
|
||||
# query internalization
|
||||
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path ../../ctx-to-lora/train_outputs/runs/Sep29_14-42-46_slurm0-a3nodeset-9_88483_1e7bb34e/checkpoint-40000/pytorch_model.bin --datasets squad --split test --eval_batch_size_gen=1 --flip_ctx_inp --max_test_samples_per_ds 10
|
||||
|
||||
# replaced squad context
|
||||
WANDB_MODE=disabled uv run python run_eval.py --checkpoint_path ../../ctx-to-lora/train_outputs/runs/Sep29_14-42-46_slurm0-a3nodeset-9_88483_1e7bb34e/checkpoint-40000/pytorch_model.bin --datasets squad_assistant_ctx_no_passage --split test --max_test_samples_per_ds 10
|
||||
WANDB_MODE=disabled uv run python run_eval.py --checkpoint_path ../../ctx-to-lora/train_outputs/runs/Sep29_14-42-46_slurm0-a3nodeset-9_88483_1e7bb34e/checkpoint-40000/pytorch_model.bin --datasets squad_negative_no_passage --split test --max_test_samples_per_ds 10
|
||||
279
scripts/main_exp/eval/imagenette_eval.py
Normal file
279
scripts/main_exp/eval/imagenette_eval.py
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from argparse import Namespace
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoProcessor, Gemma3ForConditionalGeneration
|
||||
|
||||
from ctx_to_lora.model_loading import get_tokenizer
|
||||
from ctx_to_lora.modeling.ctx_encoder import PerLayerActivations
|
||||
from ctx_to_lora.modeling.hypernet import ModulatedPretrainedModel
|
||||
from ctx_to_lora.modeling.lora_layer import apply_lora_to_layers
|
||||
from ctx_to_lora.modeling.lora_merger import combine_lora
|
||||
|
||||
CLASS_NAMES = [
|
||||
"tench",
|
||||
"English springer",
|
||||
"cassette player",
|
||||
"chain saw",
|
||||
"church",
|
||||
"French horn",
|
||||
"garbage truck",
|
||||
"gas pump",
|
||||
"golf ball",
|
||||
"parachute",
|
||||
]
|
||||
CLASS_TO_INT = {name: i for i, name in enumerate(CLASS_NAMES)}
|
||||
INPUT_TXT = f"What is in this image? Choose exactly one of the following classes: {', '.join(CLASS_NAMES)}. Response with only the correct class without any other text."
|
||||
RUN_DIR = "train_outputs/runs/Oct16_02-37-04_slurm0-a3nodeset-8_94074_1d62ecb8"
|
||||
|
||||
|
||||
def _normalize_text(text: str) -> str:
|
||||
text = re.sub(r"[^a-z0-9\s]", " ", text.lower())
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _normalize_compact(text: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]", "", text.lower())
|
||||
|
||||
|
||||
def _build_alias_map():
|
||||
alias_overrides = {
|
||||
"english springer spaniel": "English springer",
|
||||
"springer spaniel": "English springer",
|
||||
"chainsaw": "chain saw",
|
||||
"dump truck": "garbage truck",
|
||||
"refuse truck": "garbage truck",
|
||||
"garbage lorry": "garbage truck",
|
||||
"fuel pump": "gas pump",
|
||||
"gas station pump": "gas pump",
|
||||
"cassette deck": "cassette player",
|
||||
"cassette recorder": "cassette player",
|
||||
"fish": "tench",
|
||||
"tench fish": "tench",
|
||||
"french horn instrument": "French horn",
|
||||
"golfball": "golf ball",
|
||||
"skydiving": "parachute",
|
||||
"parachutist": "parachute",
|
||||
}
|
||||
|
||||
alias_map = {}
|
||||
|
||||
def register(alias: str, canonical: str):
|
||||
alias = _normalize_text(alias)
|
||||
if alias:
|
||||
alias_map[alias] = canonical
|
||||
alias_map[_normalize_compact(alias)] = canonical
|
||||
|
||||
for name in CLASS_NAMES:
|
||||
register(name, name)
|
||||
register(name.replace(" ", ""), name)
|
||||
register(name.replace(" ", "-"), name)
|
||||
|
||||
for alias, canonical in alias_overrides.items():
|
||||
register(alias, canonical)
|
||||
|
||||
return alias_map
|
||||
|
||||
|
||||
CLASS_ALIAS_MAP = _build_alias_map()
|
||||
|
||||
|
||||
def pred_to_class_id(pred_txt: str) -> int:
|
||||
norm_pred = _normalize_text(pred_txt)
|
||||
compact_pred = _normalize_compact(pred_txt)
|
||||
|
||||
for alias, canonical in CLASS_ALIAS_MAP.items():
|
||||
if alias and (alias in norm_pred or alias in compact_pred):
|
||||
return CLASS_TO_INT[canonical]
|
||||
|
||||
pred_tokens = set(norm_pred.split())
|
||||
best_class = None
|
||||
best_token_hits = -1
|
||||
for name in CLASS_NAMES:
|
||||
class_tokens = set(_normalize_text(name).split())
|
||||
if class_tokens and class_tokens.issubset(pred_tokens):
|
||||
return CLASS_TO_INT[name]
|
||||
hits = sum(token in pred_tokens for token in class_tokens)
|
||||
if hits > best_token_hits:
|
||||
best_token_hits = hits
|
||||
best_class = name
|
||||
|
||||
best_ratio = -1.0
|
||||
for name in CLASS_NAMES:
|
||||
ratio = SequenceMatcher(None, norm_pred, _normalize_text(name)).ratio()
|
||||
if ratio > best_ratio:
|
||||
best_ratio = ratio
|
||||
best_class = name
|
||||
|
||||
return CLASS_TO_INT[best_class]
|
||||
|
||||
|
||||
def load_checkpoint():
|
||||
checkpoint_path = f"{RUN_DIR}/checkpoint-80000/pytorch_model.bin"
|
||||
state_dict = torch.load(checkpoint_path)
|
||||
|
||||
model = ModulatedPretrainedModel.from_state_dict(
|
||||
state_dict,
|
||||
train=False,
|
||||
base_model_kwargs=dict(attn_implementation="flash_attention_2"),
|
||||
use_flash_attn=True,
|
||||
use_sequence_packing=False, # for generation
|
||||
)
|
||||
tokenizer = get_tokenizer("google/gemma-2-2b-it")
|
||||
model.eval()
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
def load_ctx_encoder():
|
||||
model_id = "google/gemma-3-4b-it"
|
||||
ctx_model = Gemma3ForConditionalGeneration.from_pretrained(
|
||||
model_id, device_map="auto"
|
||||
).eval()
|
||||
ctx_encoder_config = Namespace(ctx_encoder_last_layer=26, keep_lm_head=True)
|
||||
ctx_model.language_model = PerLayerActivations(
|
||||
ctx_model.language_model, ctx_encoder_config
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(model_id)
|
||||
return ctx_model, processor
|
||||
|
||||
|
||||
def template_image(img, ctx_processor):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": ""}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image", "image": img},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
inputs = ctx_processor.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
tokenize=True,
|
||||
return_dict=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def get_ctx_features(ctx_inputs, ctx_encoder):
|
||||
forward_outputs = ctx_encoder(**ctx_inputs, output_hidden_states=True)
|
||||
ctx_features = torch.stack(forward_outputs.hidden_states, dim=1)
|
||||
return ctx_features
|
||||
|
||||
|
||||
def generate_loras(ctx_inputs, ctx_features):
|
||||
generated_loras, _ = model.hypernet.generate_weights(
|
||||
ctx_features, attn_mask=torch.ones_like(ctx_inputs["input_ids"])
|
||||
)
|
||||
generated_loras = combine_lora(
|
||||
generated_loras,
|
||||
n_chunks=torch.tensor((1,), device=model.device),
|
||||
lora_bias=model.hypernet.get_head_bias()
|
||||
if model.hypernet.config.use_bias
|
||||
else None,
|
||||
)
|
||||
return generated_loras
|
||||
|
||||
|
||||
def apply_loras(model, generated_loras):
|
||||
n_queries = torch.ones(1, dtype=torch.int32, device=model.device)
|
||||
|
||||
apply_lora_to_layers(
|
||||
model.base_model,
|
||||
model.hypernet.layer_indices,
|
||||
generated_loras,
|
||||
n_queries,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
model, base_tokenizer = load_checkpoint()
|
||||
ctx_encoder, ctx_processor = load_ctx_encoder()
|
||||
ds = load_dataset("frgfm/imagenette", "full_size", split="validation")
|
||||
# ds = ds.shuffle().select(range(int(0.05 * len(ds))))
|
||||
input_ids = base_tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": INPUT_TXT}],
|
||||
add_special_tokens=False,
|
||||
return_attention_mask=False,
|
||||
add_generation_prompt=True,
|
||||
return_tensors="pt",
|
||||
).to(model.device)
|
||||
|
||||
preds = []
|
||||
pred_txts = []
|
||||
corrects = []
|
||||
labels = ds["label"]
|
||||
for sample in tqdm(ds):
|
||||
img = sample["image"]
|
||||
ctx_inputs = template_image(img, ctx_processor).to(ctx_encoder.device)
|
||||
ctx_features = get_ctx_features(ctx_inputs, ctx_encoder)
|
||||
generated_loras = generate_loras(ctx_inputs, ctx_features)
|
||||
apply_loras(model, generated_loras)
|
||||
|
||||
model_outputs = model.base_model.generate(
|
||||
input_ids, max_new_tokens=256, do_sample=False
|
||||
)
|
||||
pred_txt = base_tokenizer.decode(
|
||||
model_outputs[0][len(input_ids[0]) :], skip_special_tokens=True
|
||||
)
|
||||
|
||||
pred_txts.append(pred_txt)
|
||||
preds.append(pred_to_class_id(pred_txt))
|
||||
is_correct = preds[-1] == labels[len(preds) - 1]
|
||||
corrects.append(is_correct)
|
||||
print(
|
||||
f"GT: {CLASS_NAMES[labels[len(preds) - 1]]}, Pred: {pred_txt} -> {CLASS_NAMES[preds[-1]]}, Correct: {is_correct}"
|
||||
)
|
||||
|
||||
acc = sum(corrects) / len(corrects)
|
||||
print(f"Final accuracy: {acc:4f}")
|
||||
|
||||
jsonl_path = os.path.join(RUN_DIR, "imagenette_eval.jsonl")
|
||||
meta_path = os.path.join(RUN_DIR, "imagenette_eval.meta.json")
|
||||
|
||||
with open(jsonl_path, "w") as f:
|
||||
for i, (pred_txt, pred_id, label_id) in enumerate(
|
||||
zip(pred_txts, preds, labels)
|
||||
):
|
||||
f.write(
|
||||
json.dumps(
|
||||
{
|
||||
"index": i,
|
||||
"label": int(label_id),
|
||||
"label_name": CLASS_NAMES[label_id],
|
||||
"pred_text": pred_txt,
|
||||
"pred_class_id": int(pred_id),
|
||||
"pred_class_name": CLASS_NAMES[pred_id],
|
||||
"correct": bool(pred_id == label_id),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
meta = {
|
||||
"dataset": "frgfm/imagenette",
|
||||
"subset": "full_size",
|
||||
"split": "validation",
|
||||
"run_dir": RUN_DIR,
|
||||
"prompt": INPUT_TXT,
|
||||
"accuracy": float(acc),
|
||||
"num_samples": len(preds),
|
||||
"class_names": CLASS_NAMES,
|
||||
}
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta, f, indent=2)
|
||||
|
||||
print(f"Wrote samples to {jsonl_path}")
|
||||
print(f"Wrote metadata to {meta_path}")
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
for dataset in squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e; do
|
||||
for rate in 0.9 0.8 0.6 0.4 0.2 0.1; do
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets "$dataset" --split test --eval_batch_size_gen=1 --use_llmlingua --llmlingua_compression_rate "$rate" --truncate_if_too_long_ctx --max_test_samples_per_ds 10
|
||||
done
|
||||
done
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
# download t2l checkpoint
|
||||
uv run huggingface-cli download SakanaAI/text-to-lora --local-dir . --include "trained_t2l/gemma_2b_t2l"
|
||||
|
||||
WANDB_MODE=disabled uv run run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e --split test --eval_batch_size_gen=1 --use_t2l --max_test_samples_per_ds 10
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
# download fineweb_edu to `data/raw_datasets/fineweb_edu
|
||||
uv run data/download_fineweb_edu.py
|
||||
|
||||
# generate qa data
|
||||
# run from 000 to 013
|
||||
for shard_id in $(seq -f "%03g" 0 1); do
|
||||
uv run data/generate_fw_edu_qa_v2.py --shard_pattern "${shard_id}_00000" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it --max_length=2000 --max_model_length=2048 --debug
|
||||
uv run data/generate_fw_edu_qa_v2_repeat.py --shard_pattern "min_0_to_2000/${shard_id}*level_0" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it --debug
|
||||
|
||||
# self-generated response QA data
|
||||
uv run data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --glob_pattern "data/raw_datasets/fw_qa_v2/min_0_to_2000/${shard_id}*_level_1*" --closed_qa_prob 1.0 --debug
|
||||
done
|
||||
|
||||
|
||||
# val split
|
||||
uv run data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --glob_pattern 'data/raw_datasets/fw_qa_v2/min_0_to_2000/*_level_0_val.parquet' --debug
|
||||
|
||||
# self-gen data for other ds
|
||||
uv run data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --ds_names squad_compact ropes_compact drop_compact --split train --closed_qa_prob 1.0 --debug
|
||||
uv run data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --ds_names pwc_compact --split train --closed_qa_prob 0.0 --debug
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
#!/bin/bash
|
||||
#SBATCH --job-name=selfgen
|
||||
#SBATCH --time=5-00:00
|
||||
#SBATCH --partition=a3
|
||||
#SBATCH --ntasks-per-node=1
|
||||
#SBATCH --nodes=1
|
||||
#SBATCH --gres=gpu:1
|
||||
#SBATCH --output=slurm_logs/%x-%A_%a.out
|
||||
#SBATCH --error=slurm_logs/%x-%A_%a.err
|
||||
#SBATCH --cpus-per-task=16
|
||||
|
||||
# Collect matching files into an array (bash will expand the glob)
|
||||
files=(data/raw_datasets/fw_qa_v2/min_0_to_2000/{000..013}*_level_1.parquet)
|
||||
|
||||
# Default to 0 if SLURM_ARRAY_TASK_ID is unset
|
||||
idx=${SLURM_ARRAY_TASK_ID:-0}
|
||||
|
||||
# Bounds check
|
||||
if (( idx < 0 || idx >= ${#files[@]} )); then
|
||||
echo "Error: SLURM_ARRAY_TASK_ID $idx out of range (0..$((${#files[@]}-1)))." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
selected_file=${files[$idx]}
|
||||
|
||||
echo "Processing file index $idx: $selected_file"
|
||||
|
||||
uv run data/self_generate_qa.py \
|
||||
--vllm_model mistralai/Mistral-7B-Instruct-v0.2 \
|
||||
--glob_pattern "$selected_file" \
|
||||
--closed_qa_prob 1.0 \
|
||||
--max_new_tokens 1024
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
#!/bin/bash
|
||||
#SBATCH --job-name=selfgen
|
||||
#SBATCH --time=5-00:00
|
||||
#SBATCH --partition=a3
|
||||
#SBATCH --ntasks-per-node=1
|
||||
#SBATCH --nodes=1
|
||||
#SBATCH --gres=gpu:1
|
||||
#SBATCH --output=slurm_logs/%x-%A_%a.out
|
||||
#SBATCH --error=slurm_logs/%x-%A_%a.err
|
||||
#SBATCH --cpus-per-task=16
|
||||
|
||||
# Collect matching files into an array (bash will expand the glob)
|
||||
files=(data/raw_datasets/fw_qa_v2/min_0_to_2000/{000..013}*_level_1.parquet)
|
||||
|
||||
# Default to 0 if SLURM_ARRAY_TASK_ID is unset
|
||||
idx=${SLURM_ARRAY_TASK_ID:-0}
|
||||
|
||||
# Bounds check
|
||||
if (( idx < 0 || idx >= ${#files[@]} )); then
|
||||
echo "Error: SLURM_ARRAY_TASK_ID $idx out of range (0..$((${#files[@]}-1)))." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
selected_file=${files[$idx]}
|
||||
|
||||
echo "Processing file index $idx: $selected_file"
|
||||
|
||||
uv run data/self_generate_qa.py \
|
||||
--vllm_model Qwen/Qwen3-4B-Instruct-2507 \
|
||||
--glob_pattern "$selected_file" \
|
||||
--closed_qa_prob 1.0 \
|
||||
--max_new_tokens 1024
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
port=29051
|
||||
|
||||
uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
|
||||
--num_processes=8 --gpu_ids all train.py \
|
||||
configs/main_exp/mistral/self_gen_lv1_closed_qa_1_l2l.yaml \
|
||||
--model_name_or_path=mistralai/Mistral-7B-Instruct-v0.2 \
|
||||
--target_modules=down_proj --lora_r=8 \
|
||||
--eval_strategy=no --max_qas_len=2048 --max_qas_per_sample=1 \
|
||||
--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
|
||||
--max_steps=80000 --gradient_accumulation_steps=8 --max_packed_inp_len=4096 \
|
||||
--max_packed_ctx_len=4096 --use_per_ctx_average_loss=True --use_kl_loss=True \
|
||||
--quantize_ctx_encoder=True
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#!/bin/bash
|
||||
#SBATCH --job-name=ctxlora
|
||||
#SBATCH --nodes=1
|
||||
#SBATCH --partition=a3
|
||||
#SBATCH --gpus=8
|
||||
#SBATCH --output=slurm_logs/%x-%j.out
|
||||
#SBATCH --error=slurm_logs/%x-%j.out
|
||||
|
||||
port=$((10000 + ($SLURM_JOBID % 50000)))
|
||||
echo "Using port: $port"
|
||||
|
||||
# port=29051
|
||||
|
||||
uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
|
||||
--num_processes=8 --gpu_ids all train.py \
|
||||
configs/main_exp/qwen/self_gen_lv1_closed_qa_1_l2l.yaml \
|
||||
--model_name_or_path=Qwen/Qwen3-4B-Instruct-2507 \
|
||||
--target_modules=down_proj --lora_r=8 \
|
||||
--eval_strategy=no --max_qas_len=1024 --max_qas_per_sample=1 \
|
||||
--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
|
||||
--max_steps=80000 --gradient_accumulation_steps=16 --max_packed_inp_len=2048 \
|
||||
--max_packed_ctx_len=2048 --use_per_ctx_average_loss=True --use_kl_loss=True \
|
||||
--quantize_ctx_encoder=True \
|
||||
"$@"
|
||||
|
|
@ -4,11 +4,12 @@ port=29051
|
|||
|
||||
uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
|
||||
--num_processes=8 --gpu_ids all train.py \
|
||||
configs/main_exp/qwen/self_gen_lv1_closed_qa_1_l2l.yaml \
|
||||
--model_name_or_path=Qwen/Qwen3-4B-Instruct-2507 \
|
||||
configs/main_exp/self_gen_lv1_closed_qa_1_no_qa_l2l.yaml \
|
||||
--model_name_or_path=google/gemma-2-2b-it \
|
||||
--target_modules=down_proj --lora_r=8 \
|
||||
--eval_strategy=no --max_qas_len=2048 --max_qas_per_sample=1 \
|
||||
--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
|
||||
--max_steps=80000 --gradient_accumulation_steps=8 --max_packed_inp_len=4096 \
|
||||
--max_packed_ctx_len=4096 --use_per_ctx_average_loss=True --use_kl_loss=True \
|
||||
--quantize_ctx_encoder=True
|
||||
--quantize_ctx_encoder=True \
|
||||
"$@"
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
WANDB_MODE=disabled run uv run train.py \
|
||||
#!/bin/bash
|
||||
WANDB_MODE=disabled uv run train.py \
|
||||
configs/niah_exp/ctx_magic_number_32_256.yaml \
|
||||
--model_name_or_path=google/gemma-2-2b-it \
|
||||
--num_train_epochs=1 \
|
||||
|
|
@ -37,4 +38,5 @@
|
|||
--max_val_samples_per_ds=100 \
|
||||
--seed=1 \
|
||||
--use_per_ctx_average_loss=True \
|
||||
--torch_empty_cache_steps=10
|
||||
--torch_empty_cache_steps=10 \
|
||||
"$@"
|
||||
|
|
@ -1 +1 @@
|
|||
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path CHECKPOINT_PATH --datasets ctx_magic_number_32_1024 ctx_magic_number_1024_2048 ctx_magic_number_2048_3072 ctx_magic_number_3072_4096 ctx_magic_number_4096_5120 ctx_magic_number_5120_6144 ctx_magic_number_6144_7168 ctx_magic_number_7168_8192 ctx_magic_number_8192_9216 ctx_magic_number_9216_10240 ctx_magic_number_10240_11264 ctx_magic_number_11264_12288 ctx_magic_number_12288_13312 ctx_magic_number_13312_14336 ctx_magic_number_14336_15360 ctx_magic_number_15360_16384 ctx_magic_number_16384_20480 ctx_magic_number_20480_24576 ctx_magic_number_24576_28672 ctx_magic_number_28672_32768 ctx_magic_number_32768_40960 ctx_magic_number_40960_49152 ctx_magic_number_49152_57344 ctx_magic_number_57344_65536 ctx_magic_number_65536_73728 ctx_magic_number_73728_81920 ctx_magic_number_81920_90112 ctx_magic_number_90112_98304 ctx_magic_number_98304_106496 ctx_magic_number_106496_114688 ctx_magic_number_114688_122880 ctx_magic_number_122880_131072 --max_ctx_chunk_len=1024 --split test --eval_batch_size_gen=4
|
||||
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path $CHECKPOINT_PATH --datasets ctx_magic_number_32_1024 ctx_magic_number_1024_2048 ctx_magic_number_2048_3072 ctx_magic_number_3072_4096 ctx_magic_number_4096_5120 ctx_magic_number_5120_6144 ctx_magic_number_6144_7168 ctx_magic_number_7168_8192 ctx_magic_number_8192_9216 ctx_magic_number_9216_10240 ctx_magic_number_10240_11264 ctx_magic_number_11264_12288 ctx_magic_number_12288_13312 ctx_magic_number_13312_14336 ctx_magic_number_14336_15360 ctx_magic_number_15360_16384 ctx_magic_number_16384_20480 ctx_magic_number_20480_24576 ctx_magic_number_24576_28672 ctx_magic_number_28672_32768 ctx_magic_number_32768_40960 ctx_magic_number_40960_49152 ctx_magic_number_49152_57344 ctx_magic_number_57344_65536 ctx_magic_number_65536_73728 ctx_magic_number_73728_81920 ctx_magic_number_81920_90112 ctx_magic_number_90112_98304 ctx_magic_number_98304_106496 ctx_magic_number_106496_114688 ctx_magic_number_114688_122880 ctx_magic_number_122880_131072 --max_ctx_chunk_len=1024 --split test --eval_batch_size_gen=4
|
||||
|
|
@ -418,6 +418,10 @@ class CtxTrainingArguments:
|
|||
default=0.0,
|
||||
metadata={"help": "L1 regularization coefficient for generated LoRAs."},
|
||||
)
|
||||
add_negative_prompt: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "Whether to add negative prompt training."},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -162,9 +162,51 @@ def filter_none(sample):
|
|||
return True
|
||||
|
||||
|
||||
def add_negative_prompt_fn(samples):
|
||||
unique_contexts = set()
|
||||
ctxs, prompts, responses = [], [], []
|
||||
for ctx, prompt, response in zip(
|
||||
samples["context"], samples["prompt"], samples["response"]
|
||||
):
|
||||
if ctx in unique_contexts:
|
||||
continue
|
||||
unique_contexts.add(ctx)
|
||||
ctxs.append(ctx)
|
||||
prompts.append(prompt)
|
||||
responses.append(response)
|
||||
|
||||
logger.debug("Adding negative prompt...")
|
||||
logger.debug(f"# unique contexts: {len(unique_contexts)}")
|
||||
|
||||
# remove one last sample if the number of samples is odd
|
||||
if len(ctxs) % 2 != 0:
|
||||
ctxs.pop()
|
||||
prompts.pop()
|
||||
responses.pop()
|
||||
|
||||
# to make sure that the negative prompt/response is not the same as the original
|
||||
indices = list(np.random.permutation(len(ctxs))) + list(
|
||||
np.random.permutation(len(ctxs))
|
||||
)
|
||||
neg_ctxs, neg_prompts, neg_responses = [], [], []
|
||||
for idx in range(0, len(indices), 2):
|
||||
i = indices[idx]
|
||||
j = indices[idx + 1]
|
||||
neg_ctxs.append(ctxs[i])
|
||||
neg_prompts.append(ctxs[j] + "\n\n" + prompts[j])
|
||||
neg_responses.append(responses[j])
|
||||
|
||||
return dict(
|
||||
context=neg_ctxs + samples["context"],
|
||||
prompt=neg_prompts + samples["prompt"],
|
||||
response=neg_responses + samples["response"],
|
||||
)
|
||||
|
||||
|
||||
def load_and_process_dataset(
|
||||
ds_name: str,
|
||||
split: str,
|
||||
add_negative_prompt: bool,
|
||||
num_proc: int,
|
||||
remove_cols: bool = True,
|
||||
):
|
||||
|
|
@ -198,6 +240,13 @@ def load_and_process_dataset(
|
|||
batched=False,
|
||||
num_proc=16,
|
||||
)
|
||||
if add_negative_prompt and "context" in ds:
|
||||
ds = ds.map(
|
||||
add_negative_prompt_fn,
|
||||
batched=True,
|
||||
batch_size=100_000,
|
||||
# num_proc=num_proc,
|
||||
)
|
||||
return ds
|
||||
|
||||
|
||||
|
|
@ -215,6 +264,7 @@ def get_tokenized_dataset(
|
|||
num_chunk_probs: dict[int, float] | None,
|
||||
max_ctx_chunk_num: int,
|
||||
add_ctx_to_chat: bool,
|
||||
add_negative_prompt: bool,
|
||||
use_kl_loss: bool,
|
||||
max_new_tokens: int = 256,
|
||||
add_self_distill_template: bool = False,
|
||||
|
|
@ -233,6 +283,7 @@ def get_tokenized_dataset(
|
|||
load_and_process_kwargs = dict(
|
||||
ds_name=ds_name,
|
||||
split=split,
|
||||
add_negative_prompt=add_negative_prompt,
|
||||
)
|
||||
tokenize_kwargs = dict(
|
||||
max_qas_len=max_qas_len,
|
||||
|
|
|
|||
|
|
@ -796,6 +796,7 @@ def evaluate(
|
|||
ctx_distill_kwargs["q_model"] = q_model
|
||||
ctx_distill_kwargs["q_tokenizer"] = q_tokenizer
|
||||
ctx_distill_kwargs["q_gen_rounds"] = args.q_gen_rounds
|
||||
ctx_distill_kwargs["batch_size"] = args.cd_batch_size
|
||||
model = CtxDistillModel(peft_model, **ctx_distill_kwargs)
|
||||
|
||||
add_tracker(model._distill_context, "distill_context")
|
||||
|
|
@ -856,6 +857,7 @@ def evaluate(
|
|||
ctx_model_max_len=ctx_model_max_len,
|
||||
ctx_tokenizer=ctx_tokenizer,
|
||||
add_ctx_to_chat=add_ctx_to_chat,
|
||||
add_negative_prompt=False,
|
||||
use_kl_loss=False,
|
||||
max_new_tokens=max_new_tokens,
|
||||
set_format="pt",
|
||||
|
|
@ -1025,6 +1027,7 @@ def run_eval(
|
|||
cd_update_iterations: int = 10,
|
||||
cd_use_gen_q: bool = False,
|
||||
q_gen_rounds: int = 4,
|
||||
cd_batch_size: int = 16,
|
||||
use_iterative_mode: bool = False,
|
||||
use_llmlingua: bool = False,
|
||||
llmlingua_compression_rate: float = 0.9,
|
||||
|
|
@ -1108,6 +1111,7 @@ def run_eval(
|
|||
args.cd_update_iterations = cd_update_iterations
|
||||
args.cd_use_gen_q = cd_use_gen_q
|
||||
args.q_gen_rounds = q_gen_rounds
|
||||
args.cd_batch_size = cd_batch_size
|
||||
if use_llmlingua:
|
||||
args.use_llmlingua = use_llmlingua
|
||||
args.llmlingua_compression_rate = llmlingua_compression_rate
|
||||
|
|
|
|||
|
|
@ -195,6 +195,7 @@ class CtxDistillModel(nn.Module):
|
|||
lora_save_dir: str | None = None,
|
||||
save_after_distill: bool = True,
|
||||
q_gen_rounds: int = 4,
|
||||
batch_size: int = 16,
|
||||
):
|
||||
super().__init__()
|
||||
self.register_module("base_model", base_model)
|
||||
|
|
@ -224,7 +225,7 @@ class CtxDistillModel(nn.Module):
|
|||
log_num_train_params(self.base_model)
|
||||
self._init_optim()
|
||||
# Mini-batch size for distillation updates
|
||||
self.batch_size = 20
|
||||
self.batch_size = batch_size
|
||||
|
||||
@property
|
||||
def generation_config(self):
|
||||
|
|
|
|||
1
train.py
1
train.py
|
|
@ -280,6 +280,7 @@ def main():
|
|||
ctx_model_max_len=ctx_model_max_len,
|
||||
ctx_tokenizer=ctx_tokenizer,
|
||||
add_ctx_to_chat=add_ctx_to_chat,
|
||||
add_negative_prompt=ctx_args.add_negative_prompt,
|
||||
max_ctx_chunk_len=ctx_args.max_ctx_chunk_len,
|
||||
min_ctx_chunk_len=ctx_args.min_ctx_chunk_len,
|
||||
num_chunk_probs=ctx_args.num_chunk_probs,
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
{
|
||||
"alpha_pattern": {},
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "google/gemma-2-2b-it",
|
||||
"bias": "none",
|
||||
"corda_config": null,
|
||||
"eva_config": null,
|
||||
"exclude_modules": null,
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": false,
|
||||
"init_lora_weights": true,
|
||||
"layer_replication": null,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"loftq_config": {},
|
||||
"lora_alpha": 16,
|
||||
"lora_bias": false,
|
||||
"lora_dropout": 0.05,
|
||||
"megatron_config": null,
|
||||
"megatron_core": "megatron.core",
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"r": 8,
|
||||
"rank_pattern": {},
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"q_proj",
|
||||
"v_proj"
|
||||
],
|
||||
"task_type": "CAUSAL_LM",
|
||||
"trainable_token_indices": null,
|
||||
"use_dora": false,
|
||||
"use_rslora": true
|
||||
}
|
||||
|
|
@ -1,875 +0,0 @@
|
|||
additional_eval_descs:
|
||||
- dogs;cats;bananas;
|
||||
- 7@9.qwepra#/.sd,s'2OC^039u#rdagjbL
|
||||
- ggggggggggggggggggggg
|
||||
also_val_on_train: false
|
||||
autoreg_gen: false
|
||||
batch_size: 4
|
||||
config: null
|
||||
debug: false
|
||||
delta_w_scaling: 10000
|
||||
emb_model: Alibaba-NLP/gte-large-en-v1.5
|
||||
encoder_type: linear
|
||||
epochs: 20000
|
||||
equally_weight_sample: true
|
||||
eval_ds_info:
|
||||
arc_challenge:
|
||||
descriptions:
|
||||
- This task is about analyzing questions which examine your grasp of scientific
|
||||
ideas. You must connect conceptual knowledge with practical examples from geology,
|
||||
ecology and environmental changes.
|
||||
- The objective here is to evaluate various scientific scenarios and infer the
|
||||
most logical explanations or definitions based on established knowledge. This
|
||||
task will strengthen your analytical and reasoning skills in the context of
|
||||
natural science.
|
||||
- Your role is to interpret questions focusing on earth science and biological
|
||||
interactions. This demands a clear understanding of relevant processes, such
|
||||
as decomposition, weathering, and species adaptation.
|
||||
arc_easy:
|
||||
descriptions:
|
||||
- Your job is to discern which information best answers a posed question, focusing
|
||||
on practical examples and scientific principles. This requires a strong grasp
|
||||
of underlying concepts in ecology or physics.
|
||||
- You will analyze questions that explore important connections such as environmental
|
||||
issues or animal adaptations. Utilize your background knowledge to evaluate
|
||||
and select the most fitting answer.
|
||||
- This task involves selecting answers that reflect accurate relationships or
|
||||
effects seen in nature or society. You will need to sort through potential choices
|
||||
critically to find the appropriate one.
|
||||
boolq:
|
||||
descriptions:
|
||||
- Analyze the given details about various subjects, including movies, sports,
|
||||
and television shows. Your role is to confirm whether certain claims are true
|
||||
or false.
|
||||
- Your task is to determine the truthfulness of specific statements based on the
|
||||
provided background information. This requires careful reading and comprehension
|
||||
of the content.
|
||||
- The goal is to evaluate factual claims made in relation to highlighted texts.
|
||||
You will need to discern whether the statements align with the information provided.
|
||||
gsm8k:
|
||||
descriptions:
|
||||
- You will be tasked with interpreting mathematical situations described in words.
|
||||
The goal is to use logical reasoning and calculations to determine the numerical
|
||||
answers based on the context provided.
|
||||
- This task challenges your problem-solving abilities through mathematical reasoning.
|
||||
You must carefully read each scenario and systematically work through the data
|
||||
to compute the final outcome.
|
||||
- Your role is to engage with practical math scenarios presented as questions.
|
||||
The task requires translating textual data into numerical operations that will
|
||||
lead you to the final solution.
|
||||
hellaswag:
|
||||
descriptions:
|
||||
- This task revolves around completing an unfinished text by selecting an ending
|
||||
that matches its tone and context. It requires you to think critically about
|
||||
how narratives develop and conclude effectively.
|
||||
- This task asks you to select a suitable conclusion for an unfinished narrative
|
||||
or instructional content. It tests your comprehension and reasoning skills as
|
||||
you assess how well each option aligns with the given text.
|
||||
- Your task involves completing an incomplete passage by selecting the ending
|
||||
that logically continues the context provided. This requires reading comprehension
|
||||
and the ability to infer meaning from a text.
|
||||
humaneval:
|
||||
descriptions:
|
||||
- Engage in building distinct functions that meet the requirements of various
|
||||
presented problems, honing your ability to translate problem statements into
|
||||
logical code. Utilize structured thinking to implement efficient solutions.
|
||||
- You are tasked with generating specific solutions in Python by interpreting
|
||||
problem descriptions associated with tasks like counting odds or validating
|
||||
inputs. Recognizing patterns and leveraging programming techniques will be beneficial.
|
||||
- This task focuses on developing algorithms in Python for specific scenarios,
|
||||
such as counting characters, assessing conditions between numbers, or converting
|
||||
integers into a different format. Critical thinking and algorithmic design will
|
||||
be important.
|
||||
lol_035:
|
||||
descriptions:
|
||||
- Your job is to transform a fill-in-the-blank question by changing just one key
|
||||
word or phrase, allowing for a new but coherent answer. This requires analytical
|
||||
thinking about vocabulary and context.
|
||||
- You need to innovate a way to adjust an existing question so it accepts a different
|
||||
name as an answer. This process enhances your skills in vocabulary rich expressions
|
||||
and logical reasoning.
|
||||
- The objective is to carefully edit questions such that the identity of the answer
|
||||
changes while keeping most statements intact. It emphasizes the importance of
|
||||
word choice and its impact on meaning.
|
||||
lol_039:
|
||||
descriptions:
|
||||
- This task is about spotting keywords that appear in both sentences. You need
|
||||
to focus on meaningful words while ensuring not to include common stop words.
|
||||
- The goal here is to extract important overlapping vocabulary from two related
|
||||
sentences. This requires critical thinking and familiarity with different word
|
||||
derivatives.
|
||||
- Your objective is to analyze two sentences and find words that are shared between
|
||||
them. This requires insight into language nuances and the ability to discern
|
||||
significant terms.
|
||||
lol_084:
|
||||
descriptions:
|
||||
- This task involves assessing a passage with numbered facts about people's movements.
|
||||
You need to find and select the fact that clearly indicates the location of
|
||||
a specified person based on the context provided.
|
||||
- Your task is to read a set of facts within a passage and determine which fact
|
||||
provides the necessary information to answer a question about the location of
|
||||
a specific person.
|
||||
- You will evaluate enumerated facts and a question regarding an individual's
|
||||
whereabouts. The goal is to isolate the fact that supplies essential information
|
||||
for responding accurately to the question.
|
||||
lol_1198:
|
||||
descriptions:
|
||||
- You need to evaluate whether an action committed by PersonX causes PersonY or
|
||||
others to want something indicated in another phrase. It requires you to think
|
||||
critically about relationships and emotional connections.
|
||||
- Your job is to interpret short sequences of events. For each sequence, decide
|
||||
if the first part creates a desire for the second part. Familiarity with human
|
||||
behaviors and desires will be crucial in making accurate assessments.
|
||||
- In this task, you consider two elements where one may lead to a desire for another.
|
||||
Evaluating these connections requires insight into motivations behind human
|
||||
behavior and how actions affect relationships.
|
||||
lol_140:
|
||||
descriptions:
|
||||
- In this challenge, you will consider a prompt and its accompanying completions,
|
||||
aiming to choose the response that best fits the established style of the prompt.
|
||||
Pay attention to informal versus formal tones.
|
||||
- The task involves evaluating two responses based on their alignment with the
|
||||
style of a given prompt. You need to identify which response mirrors the tone
|
||||
and language used in the prompt.
|
||||
- This activity involves selecting between two responses based on their similarity
|
||||
in style to a prompt. Evaluate tone, word choice, and sentence flow to find
|
||||
your answer.
|
||||
lol_1448:
|
||||
descriptions:
|
||||
- You will be presented with different sentences that include medical information.
|
||||
The challenge is to recognize and state the name of the disease or disorder
|
||||
featured in each sentence.
|
||||
- Your task is to read a provided sentence and identify the name of a disorder
|
||||
or disease mentioned within it. This requires familiarity with medical terminology
|
||||
and conditions.
|
||||
- Your job is to decipher sentences containing descriptions of health conditions.
|
||||
From these, you must identify and return the appropriate title of the disorder
|
||||
or disease.
|
||||
lol_1557:
|
||||
descriptions:
|
||||
- In this activity, you will identify mistakes in given English sentences and
|
||||
produce a version that is free from errors. Attention to detail is important
|
||||
in ensuring clarity and correctness.
|
||||
- The goal here is to carefully analyze flawed sentences and create polished versions.
|
||||
Strong language skills are needed to spot errors and make appropriate corrections.
|
||||
- This task involves revising sentences that contain mistakes in grammar or style.
|
||||
Your ability to recognize these issues and rewrite them correctly will be key.
|
||||
lol_1711:
|
||||
descriptions:
|
||||
- In this task, you need to write a short poem inspired by a title. The focus
|
||||
is on capturing the voice of a young child, using straightforward language and
|
||||
continuous sentences.
|
||||
- This task requires you to generate a poem related to a provided title. Think
|
||||
like a child, keeping the language simple and letting your thoughts flow freely
|
||||
without stopping for punctuation.
|
||||
- You will create a light-hearted poem related to a given title. It should be
|
||||
straightforward and fun, reflecting the spontaneous and imaginative ideas of
|
||||
young children through unpunctuated lines.
|
||||
lol_202:
|
||||
descriptions:
|
||||
- This task requires critical thinking as you examine a statement and three choices.
|
||||
You are to find the sentence that represents a disagreement with what is stated,
|
||||
showing an understanding of contrasting ideas.
|
||||
- In this task, you will evaluate a statement and three possible responses. Your
|
||||
objective is to identify which response presents an opposing viewpoint or argument
|
||||
to the original statement provided.
|
||||
- In this exercise, you will encounter a statement and three accompanying sentences.
|
||||
Your role is to isolate the sentence that most directly contradicts the premise
|
||||
of the original statement, requiring strong analytical skills.
|
||||
lol_275:
|
||||
descriptions:
|
||||
- The assignment challenges you to rethink sentence structures based on targeted
|
||||
changes. Your ability to navigate aspects like voice and number will be essential
|
||||
in achieving the desired modifications accurately.
|
||||
- "In this activity, you\u2019ll be tasked with modifying provided text based\
|
||||
\ on specific rules concerning tense, number, or voice. You need to apply linguistic\
|
||||
\ knowledge creatively while ensuring coherence within your changes."
|
||||
- "You\u2019re challenged to change sentences in particular ways, such as altering\
|
||||
\ verb tenses or noun numbers. It requires careful attention to detail in understanding\
|
||||
\ how these changes affect the overall structure of the sentences."
|
||||
lol_304:
|
||||
descriptions:
|
||||
- You are tasked with identifying what the marked number refers to in a conversational
|
||||
excerpt. It requires logical reasoning and an understanding of situational cues
|
||||
present in the text.
|
||||
- Engage with dialogues to uncover what meaning lies behind an underlined number.
|
||||
Recognizing references, ages, or other implied categories based on conversational
|
||||
context is essential in this task.
|
||||
- This task focuses on determining the definition or representation of a highlighted
|
||||
number within text-based interactions. Understanding context and relationships
|
||||
between elements is crucial for success.
|
||||
lol_362:
|
||||
descriptions:
|
||||
- Your role is to analyze a conversation and select the response that effectively
|
||||
embraces the idea of collaboration in humor. Look for the one that acknowledges
|
||||
the previous statement and adds more to the narrative.
|
||||
- This task involves interpreting a given prompt from an improv comedy perspective.
|
||||
You must identify which of two responses aligns with the principle of agreeing
|
||||
and expanding on a topic to keep the dialogue flowing.
|
||||
- This task challenges you to determine which response exemplifies a key improvisational
|
||||
concept by accepting what has already been stated and taking it in a new direction,
|
||||
contributing to story development or humor.
|
||||
lol_614:
|
||||
descriptions:
|
||||
- You are tasked with dissecting a short story by focusing on one key sentence.
|
||||
Reflect on prior occurrences in the narrative to see how they connect as causes
|
||||
or enabling factors for that moment in the plot.
|
||||
- Your objective is to analyze a chosen sentence within a story. You should consider
|
||||
prior events that may have directly led to this moment, creating connections
|
||||
that showcase cause-and-effect dynamics within the narrative flow.
|
||||
- The task here is to interpret a brief narrative by selecting a particular line
|
||||
for examination. Assess what happens before this point to establish whether
|
||||
any prior actions contribute as causes or simply set the stage.
|
||||
lol_636:
|
||||
descriptions:
|
||||
- You will be working on a list that includes both numerical values and lowercase
|
||||
alphabets. The focus is on filtering out the alphabets, ensuring there are no
|
||||
repeats, and sorting them in alphabetical order.
|
||||
- In this task, you need to work with a list that contains both numbers and letters.
|
||||
Your goal is to find and organize the letters in alphabetical order, ensuring
|
||||
that only unique letters are included.
|
||||
- This task involves analyzing a list comprised of various items, including digits
|
||||
and lowercase letters. You must identify the letters, sort them without duplicates,
|
||||
and prepare them for output.
|
||||
lol_701:
|
||||
descriptions:
|
||||
- In this exercise, you will encounter various computer science-related scenarios.
|
||||
Assess your understanding of programming operations, code functionality, and
|
||||
data management to provide accurate answers.
|
||||
- You're tasked with answering questions that assess your understanding of high
|
||||
school computer science topics. Leverage your familiarity with programming languages,
|
||||
computer security, and data structures as you provide your response.
|
||||
- This task involves interpreting computer science questions and applying relevant
|
||||
concepts to arrive at a correct answer. You should focus on key ideas in programming
|
||||
and system functionality.
|
||||
lol_705:
|
||||
descriptions:
|
||||
- The objective of this task is to evaluate your understanding of various macroeconomic
|
||||
principles. You will analyze questions about fiscal policies, money circulation,
|
||||
and market behaviors to derive the right conclusions.
|
||||
- You will explore high school-level macroeconomic topics through a series of
|
||||
questions. Your job is to identify the correct assertions that align with your
|
||||
comprehension of economic functions and behavior.
|
||||
- This task involves critically thinking about macroeconomic concepts and their
|
||||
implications. You need to interpret various dynamics like aggregate demand and
|
||||
fiscal measures while identifying the most appropriate answer.
|
||||
lol_706:
|
||||
descriptions:
|
||||
- Your task is to apply your knowledge of high school mathematics to solve various
|
||||
problems, requiring logical reasoning and an understanding of mathematical principles.
|
||||
- This activity involves interpreting mathematical questions, translating them
|
||||
into calculations, and determining the correct outcomes through analytical thinking.
|
||||
- Delve into questions that require you to connect mathematical concepts with
|
||||
practical applications. Your task will be to pull from your math knowledge in
|
||||
order to resolve each query thoughtfully.
|
||||
lol_710:
|
||||
descriptions:
|
||||
- Your role is to navigate through a series of statistics questions that require
|
||||
practical knowledge about quantitative research methods. This includes understanding
|
||||
how different sampling strategies can influence research outcomes.
|
||||
- Engage with statistical queries that challenge your knowledge of data interpretation
|
||||
and analysis. The task entails distinguishing correct statements based on principles
|
||||
like mean comparison, sampling distributions, and experimental design.
|
||||
- In this exercise, you'll be assessing a range of assertions regarding statistical
|
||||
terms and principles. Drawing upon your background in statistics will enable
|
||||
you to identify valid conclusions from those presented.
|
||||
lol_717:
|
||||
descriptions:
|
||||
- The objective is to evaluate various scenarios presented in the form of questions
|
||||
and accurately identify the logical fallacies at play. Critical thinking skills
|
||||
are crucial for this task.
|
||||
- You are tasked with assessing hypothetical arguments for flawed logic. Being
|
||||
well-versed in identifying common fallacies will support you in this analytical
|
||||
challenge.
|
||||
- In this exercise, you analyze statements regarding logical reasoning to determine
|
||||
which fallacy is being described. This requires familiarity with various types
|
||||
of logical errors in argumentation.
|
||||
lol_726:
|
||||
descriptions:
|
||||
- Engage with questions centered around philosophical themes and thinkers. This
|
||||
requires a solid understanding of philosophical arguments and an ability to
|
||||
discern correct interpretations from the given choices.
|
||||
- Engage in a thoughtful exploration of significant philosophical questions, focusing
|
||||
on concepts put forth by renowned thinkers. Your task revolves around accurately
|
||||
identifying which interpretation is most aligned with established views.
|
||||
- You will navigate through questions that test your grasp of philosophical principles.
|
||||
The task requires critical thinking, as you analyze the provided answers to
|
||||
find the one that accurately reflects philosophical understanding.
|
||||
lol_742:
|
||||
descriptions:
|
||||
- Your goal is to read a given text and pinpoint answers related to event frequency.
|
||||
This might include identifying how many times an event has happened or how often
|
||||
it is likely to occur, based on the information provided.
|
||||
- "You\u2019ll be engaging with texts that mention different activities and assessing\
|
||||
\ how often they occur. This requires analytic thinking as you need to extract\
|
||||
\ meaningful information without extraneous details."
|
||||
- "The challenge lies in extracting succinct answers from written material regarding\
|
||||
\ how frequently various events happen. You\u2019ll need to interpret descriptions\
|
||||
\ carefully, determining the most likely answers based on context."
|
||||
mbpp:
|
||||
descriptions:
|
||||
- Your challenge is to solve a series of problems by writing functions in Python.
|
||||
These problems require handling lists and strings, allowing you to showcase
|
||||
your proficiency in coding while addressing practical programming scenarios.
|
||||
- You will be tasked with creating various Python functions that tackle programming
|
||||
challenges. The exercises will test your ability to manipulate data structures,
|
||||
search for patterns, and implement checks on numerical products.
|
||||
- The goal is to develop Python functions that perform designated operations on
|
||||
lists and strings. This requires a solid grasp of logical reasoning and the
|
||||
ability to apply relevant algorithms in your code.
|
||||
openbookqa:
|
||||
descriptions:
|
||||
- Analyze the provided statements carefully and determine which one best fits
|
||||
into the context of the passage. This requires comprehension skills and the
|
||||
ability to make logical inferences.
|
||||
- Consider each option in relation to what is presented in the input. Discern
|
||||
which one logically completes or responds accurately to the notion being expressed.
|
||||
- Here, you'll be presented with different statements, and your role is to decide
|
||||
which one appropriately complements or responds to a scenario. This process
|
||||
involves critical analysis and synthesis of information.
|
||||
piqa:
|
||||
descriptions:
|
||||
- You will explore practical questions and select an answer that presents a logical
|
||||
and widely accepted approach to solve a given problem or complete a task successfully.
|
||||
- Analyze the provided scenarios where practical advice or solutions are required,
|
||||
focusing on selecting the most commonly used or convenient method.
|
||||
- Given a question related to common tasks, your responsibility is to discern
|
||||
which proposed solution aligns with typical practices or makes the task easier
|
||||
to achieve.
|
||||
winogrande:
|
||||
descriptions:
|
||||
- In this exercise, you need to read short narratives and discern which person
|
||||
or object fits best within the context of the sentence.
|
||||
- This task requires synthesizing information from concise textual scenarios to
|
||||
identify crucial elements that drive the narrative forward.
|
||||
- The goal is to evaluate descriptions and select the entity that best aligns
|
||||
with the sentiments or actions presented in the scenario.
|
||||
exp_setup: hyper_lora
|
||||
factorized: false
|
||||
grad_accum_steps: 1
|
||||
head_in_size: 2048
|
||||
head_use_bias: false
|
||||
hypernet_latent_size: 512
|
||||
inp_max_len: 512
|
||||
keep_only_best: true
|
||||
l2_reg_generated_w: 0.001
|
||||
label_smoothing: 0.1
|
||||
learnable_AB_offset: false
|
||||
learnable_pos_emb: false
|
||||
logging_freq: 100
|
||||
lr: 2.5e-05
|
||||
max_grad_norm: 1.0
|
||||
model_dir: google/gemma-2-2b-it
|
||||
model_watch_freq: 5000
|
||||
mt_lora_path: null
|
||||
n_descs_per_ds: 128
|
||||
n_embs_per_sampled_task: null
|
||||
n_points_per_task: 1
|
||||
n_tasks_per_batch: 8
|
||||
n_train_ds: 479
|
||||
neftune_noise_alpha: 5.0
|
||||
notes: null
|
||||
pred_z_score: true
|
||||
run_name: 20250529-065318_9LnRVQHj
|
||||
save_dir: train_outputs/sft/hyper_lora/20250529-065318_9LnRVQHj
|
||||
save_freq: 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
save_to_base_model_dir: false
|
||||
seed: 42
|
||||
sft_mode: completion
|
||||
shared_AB_head: false
|
||||
skip_eval: false
|
||||
target_modules:
|
||||
- q_proj
|
||||
- v_proj
|
||||
train_ds_names:
|
||||
- lol_742
|
||||
- lol_1198
|
||||
- lol_717
|
||||
- lol_705
|
||||
- lol_275
|
||||
- lol_636
|
||||
- lol_084
|
||||
- lol_1711
|
||||
- lol_140
|
||||
- lol_1448
|
||||
- lol_453
|
||||
- lol_1207
|
||||
- lol_734
|
||||
- lol_298
|
||||
- lol_587
|
||||
- lol_703
|
||||
- lol_147
|
||||
- lol_564
|
||||
- lol_1341
|
||||
- lol_201
|
||||
- lol_890
|
||||
- lol_908
|
||||
- lol_1428
|
||||
- lol_325
|
||||
- lol_1669
|
||||
- lol_246
|
||||
- lol_357
|
||||
- lol_1518
|
||||
- lol_1148
|
||||
- lol_1605
|
||||
- lol_867
|
||||
- lol_209
|
||||
- lol_751
|
||||
- lol_161
|
||||
- lol_105
|
||||
- lol_645
|
||||
- lol_442
|
||||
- lol_075
|
||||
- lol_269
|
||||
- lol_1568
|
||||
- lol_834
|
||||
- lol_1603
|
||||
- lol_685
|
||||
- lol_083
|
||||
- lol_390
|
||||
- lol_750
|
||||
- lol_1631
|
||||
- lol_1529
|
||||
- lol_746
|
||||
- lol_1217
|
||||
- lol_725
|
||||
- lol_889
|
||||
- lol_492
|
||||
- lol_620
|
||||
- lol_294
|
||||
- lol_641
|
||||
- lol_318
|
||||
- lol_846
|
||||
- lol_316
|
||||
- lol_1188
|
||||
- lol_629
|
||||
- lol_770
|
||||
- lol_1482
|
||||
- lol_499
|
||||
- lol_955
|
||||
- lol_719
|
||||
- lol_723
|
||||
- lol_087
|
||||
- lol_211
|
||||
- lol_901
|
||||
- lol_1483
|
||||
- lol_089
|
||||
- lol_627
|
||||
- lol_153
|
||||
- lol_1342
|
||||
- lol_828
|
||||
- lol_064
|
||||
- lol_1387
|
||||
- lol_400
|
||||
- lol_1294
|
||||
- lol_243
|
||||
- lol_1572
|
||||
- lol_1151
|
||||
- lol_574
|
||||
- lol_428
|
||||
- lol_366
|
||||
- lol_926
|
||||
- lol_1503
|
||||
- lol_130
|
||||
- lol_515
|
||||
- lol_151
|
||||
- lol_619
|
||||
- lol_1562
|
||||
- lol_632
|
||||
- lol_966
|
||||
- lol_605
|
||||
- lol_1487
|
||||
- lol_707
|
||||
- lol_1379
|
||||
- lol_1489
|
||||
- lol_1567
|
||||
- lol_1384
|
||||
- lol_1404
|
||||
- lol_691
|
||||
- lol_728
|
||||
- lol_219
|
||||
- lol_964
|
||||
- lol_1509
|
||||
- lol_582
|
||||
- lol_455
|
||||
- lol_963
|
||||
- lol_382
|
||||
- lol_859
|
||||
- lol_1393
|
||||
- lol_1565
|
||||
- lol_1720
|
||||
- lol_670
|
||||
- lol_689
|
||||
- lol_324
|
||||
- lol_1420
|
||||
- lol_618
|
||||
- lol_625
|
||||
- lol_377
|
||||
- lol_929
|
||||
- lol_296
|
||||
- lol_852
|
||||
- lol_1332
|
||||
- lol_1444
|
||||
- lol_850
|
||||
- lol_708
|
||||
- lol_1292
|
||||
- lol_110
|
||||
- lol_155
|
||||
- lol_429
|
||||
- lol_245
|
||||
- lol_137
|
||||
- lol_1566
|
||||
- lol_1146
|
||||
- lol_924
|
||||
- lol_022
|
||||
- lol_118
|
||||
- lol_687
|
||||
- lol_1167
|
||||
- lol_1502
|
||||
- lol_865
|
||||
- lol_181
|
||||
- lol_698
|
||||
- lol_956
|
||||
- lol_732
|
||||
- lol_721
|
||||
- lol_370
|
||||
- lol_1400
|
||||
- lol_1199
|
||||
- lol_1606
|
||||
- lol_288
|
||||
- lol_1670
|
||||
- lol_207
|
||||
- lol_1206
|
||||
- lol_457
|
||||
- lol_1308
|
||||
- lol_1310
|
||||
- lol_874
|
||||
- lol_1541
|
||||
- lol_1609
|
||||
- lol_210
|
||||
- lol_1318
|
||||
- lol_365
|
||||
- lol_755
|
||||
- lol_123
|
||||
- lol_1316
|
||||
- lol_1378
|
||||
- lol_475
|
||||
- lol_903
|
||||
- lol_070
|
||||
- lol_720
|
||||
- lol_067
|
||||
- lol_1564
|
||||
- lol_270
|
||||
- lol_167
|
||||
- lol_1504
|
||||
- lol_178
|
||||
- lol_277
|
||||
- lol_1315
|
||||
- lol_1434
|
||||
- lol_192
|
||||
- lol_1157
|
||||
- lol_672
|
||||
- lol_563
|
||||
- lol_714
|
||||
- lol_1212
|
||||
- lol_495
|
||||
- lol_1583
|
||||
- lol_753
|
||||
- lol_343
|
||||
- lol_1427
|
||||
- lol_092
|
||||
- lol_1285
|
||||
- lol_333
|
||||
- lol_329
|
||||
- lol_398
|
||||
- lol_157
|
||||
- lol_074
|
||||
- lol_1506
|
||||
- lol_697
|
||||
- lol_285
|
||||
- lol_393
|
||||
- lol_1147
|
||||
- lol_1585
|
||||
- lol_648
|
||||
- lol_353
|
||||
- lol_1431
|
||||
- lol_148
|
||||
- lol_585
|
||||
- lol_477
|
||||
- lol_1582
|
||||
- lol_355
|
||||
- lol_633
|
||||
- lol_093
|
||||
- lol_722
|
||||
- lol_566
|
||||
- lol_1152
|
||||
- lol_1452
|
||||
- lol_925
|
||||
- lol_1703
|
||||
- lol_833
|
||||
- lol_1210
|
||||
- lol_679
|
||||
- lol_1203
|
||||
- lol_1089
|
||||
- lol_389
|
||||
- lol_378
|
||||
- lol_637
|
||||
- lol_101
|
||||
- lol_1355
|
||||
- lol_640
|
||||
- lol_344
|
||||
- lol_1190
|
||||
- lol_733
|
||||
- lol_107
|
||||
- lol_1209
|
||||
- lol_505
|
||||
- lol_1385
|
||||
- lol_1135
|
||||
- lol_328
|
||||
- lol_609
|
||||
- lol_413
|
||||
- lol_072
|
||||
- lol_1425
|
||||
- lol_1451
|
||||
- lol_713
|
||||
- lol_642
|
||||
- lol_1321
|
||||
- lol_454
|
||||
- lol_504
|
||||
- lol_696
|
||||
- lol_1429
|
||||
- lol_1645
|
||||
- lol_431
|
||||
- lol_1317
|
||||
- lol_131
|
||||
- lol_675
|
||||
- lol_1158
|
||||
- lol_1325
|
||||
- lol_1216
|
||||
- lol_1347
|
||||
- lol_1328
|
||||
- lol_630
|
||||
- lol_460
|
||||
- lol_1665
|
||||
- lol_1154
|
||||
- lol_300
|
||||
- lol_617
|
||||
- lol_1508
|
||||
- lol_628
|
||||
- lol_600
|
||||
- lol_1380
|
||||
- lol_489
|
||||
- lol_905
|
||||
- lol_065
|
||||
- lol_044
|
||||
- lol_069
|
||||
- lol_356
|
||||
- lol_403
|
||||
- lol_590
|
||||
- lol_577
|
||||
- lol_819
|
||||
- lol_1721
|
||||
- lol_351
|
||||
- lol_904
|
||||
- lol_244
|
||||
- lol_1712
|
||||
- lol_683
|
||||
- lol_1197
|
||||
- lol_1592
|
||||
- lol_274
|
||||
- lol_278
|
||||
- lol_1534
|
||||
- lol_891
|
||||
- lol_694
|
||||
- lol_497
|
||||
- lol_488
|
||||
- lol_144
|
||||
- lol_1722
|
||||
- lol_1728
|
||||
- lol_291
|
||||
- lol_284
|
||||
- lol_1288
|
||||
- lol_128
|
||||
- lol_580
|
||||
- lol_616
|
||||
- lol_1401
|
||||
- lol_716
|
||||
- lol_1421
|
||||
- lol_1656
|
||||
- lol_1311
|
||||
- lol_516
|
||||
- lol_593
|
||||
- lol_138
|
||||
- lol_1186
|
||||
- lol_1326
|
||||
- lol_119
|
||||
- lol_108
|
||||
- lol_584
|
||||
- lol_388
|
||||
- lol_045
|
||||
- lol_821
|
||||
- lol_1581
|
||||
- lol_695
|
||||
- lol_596
|
||||
- lol_568
|
||||
- lol_085
|
||||
- lol_1495
|
||||
- lol_927
|
||||
- lol_1453
|
||||
- lol_1201
|
||||
- lol_923
|
||||
- lol_1204
|
||||
- lol_1510
|
||||
- lol_754
|
||||
- lol_1403
|
||||
- lol_1192
|
||||
- lol_565
|
||||
- lol_146
|
||||
- lol_666
|
||||
- lol_050
|
||||
- lol_704
|
||||
- lol_934
|
||||
- lol_579
|
||||
- lol_1196
|
||||
- lol_267
|
||||
- lol_206
|
||||
- lol_936
|
||||
- lol_323
|
||||
- lol_494
|
||||
- lol_461
|
||||
- lol_1409
|
||||
- lol_1313
|
||||
- lol_076
|
||||
- lol_686
|
||||
- lol_740
|
||||
- lol_1211
|
||||
- lol_113
|
||||
- lol_280
|
||||
- lol_1551
|
||||
- lol_116
|
||||
- lol_518
|
||||
- lol_1520
|
||||
- lol_079
|
||||
- lol_513
|
||||
- lol_1590
|
||||
- lol_1713
|
||||
- lol_1386
|
||||
- lol_063
|
||||
- lol_183
|
||||
- lol_1447
|
||||
- lol_671
|
||||
- lol_068
|
||||
- lol_858
|
||||
- lol_699
|
||||
- lol_1593
|
||||
- lol_700
|
||||
- lol_1607
|
||||
- lol_121
|
||||
- lol_190
|
||||
- lol_1168
|
||||
- lol_195
|
||||
- lol_1723
|
||||
- lol_1449
|
||||
- lol_363
|
||||
- lol_1419
|
||||
- lol_1398
|
||||
- lol_893
|
||||
- lol_326
|
||||
- lol_1194
|
||||
- lol_102
|
||||
- lol_145
|
||||
- lol_1338
|
||||
- lol_391
|
||||
- lol_176
|
||||
- lol_319
|
||||
- lol_359
|
||||
- lol_856
|
||||
- lol_1729
|
||||
- lol_761
|
||||
- lol_1320
|
||||
- lol_1596
|
||||
- lol_1601
|
||||
- lol_615
|
||||
- lol_1283
|
||||
- lol_638
|
||||
- lol_607
|
||||
- lol_692
|
||||
- lol_588
|
||||
- lol_129
|
||||
- lol_1200
|
||||
- lol_1486
|
||||
- lol_1406
|
||||
- lol_583
|
||||
- lol_094
|
||||
- lol_1731
|
||||
- lol_308
|
||||
- lol_664
|
||||
- lol_163
|
||||
- lol_127
|
||||
- lol_736
|
||||
- lol_379
|
||||
- lol_330
|
||||
- lol_1599
|
||||
- lol_875
|
||||
- lol_1214
|
||||
- lol_933
|
||||
- lol_1319
|
||||
- lol_456
|
||||
- lol_1189
|
||||
- lol_1657
|
||||
- lol_517
|
||||
- lol_385
|
||||
- lol_472
|
||||
- lol_047
|
||||
- lol_1533
|
||||
- lol_249
|
||||
- lol_095
|
||||
- lol_1479
|
||||
- lol_724
|
||||
- lol_1418
|
||||
- lol_507
|
||||
- lol_043
|
||||
- lol_1394
|
||||
- lol_149
|
||||
- lol_1156
|
||||
- lol_1598
|
||||
- lol_322
|
||||
- lol_1714
|
||||
- lol_727
|
||||
- lol_1704
|
||||
- lol_077
|
||||
- lol_1390
|
||||
- lol_667
|
||||
- lol_1724
|
||||
- lol_1088
|
||||
- lol_1087
|
||||
- lol_550
|
||||
- lol_892
|
||||
- lol_674
|
||||
- lol_509
|
||||
- lol_346
|
||||
- lol_769
|
||||
- lol_1322
|
||||
- lol_341
|
||||
- lol_290
|
||||
- lol_335
|
||||
- lol_879
|
||||
- lol_861
|
||||
- lol_594
|
||||
- lol_066
|
||||
- lol_162
|
||||
- lol_1584
|
||||
- lol_1622
|
||||
- lol_247
|
||||
training_task: sft
|
||||
use_default_desc: false
|
||||
use_hierarchical_sampler: true
|
||||
use_hypernet: true
|
||||
use_inp_as_desc: false
|
||||
use_one_hot_task_emb: false
|
||||
use_per_sample_desc: false
|
||||
use_per_task_emb: true
|
||||
val_batch_size: 32
|
||||
val_freq: 10000
|
||||
warmup_frac: 0.2
|
||||
weight_decay: 0.001
|
||||
Loading…
Add table
Add a link
Reference in a new issue