diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png
new file mode 100644
index 0000000..0f8e0ee
Binary files /dev/null and b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png differ
diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png
new file mode 100644
index 0000000..5818be7
Binary files /dev/null and b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png differ
diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png
new file mode 100644
index 0000000..5818be7
Binary files /dev/null and b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png differ
diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png
new file mode 100644
index 0000000..5818be7
Binary files /dev/null and b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png differ
diff --git a/tmp/icl_results.py b/tmp/icl_results.py
new file mode 100644
index 0000000..94728b3
--- /dev/null
+++ b/tmp/icl_results.py
@@ -0,0 +1,204 @@
+import json
+from pathlib import Path
+
+all_results_path = Path(
+    "eval_results/google/gemma-2-2b-it/20250918-111058_734998c4/all_results.json"
+)
+all_result_modulated_path = Path(
+    "../train_outputs/runs/Aug10_08-29-46_sakura-003_2000_f73050aa/eval-results-100000/all_results.json"
+)
+with open(all_results_path) as f:
+    all_results = json.load(f)
+
+with open(all_result_modulated_path) as f:
+    all_results_modulated = json.load(f)
+
+# Find keys with test_metaicl_macro_f1 and test_metaicl_no_context_macro_f1 prefixes
+metaicl_keys = [k for k in all_results.keys() if k.startswith("test_metaicl_macro_f1")]
+no_context_keys = [
+    k for k in all_results.keys() if k.startswith("test_metaicl_no_context_macro_f1")
+]
+
+# Sort keys for consistent ordering
+metaicl_keys.sort()
+no_context_keys.sort()
+
+print("Comparison of MetaICL macro F1 scores:")
+print("=" * 140)
+print(f"{'Without Context':<40} {'With Context':<40} {'Modulated (No Context)':<40}")
+print("-" * 140)
+
+# Create a mapping of task names to values for easier comparison
+metaicl_dict = {}
+no_context_dict = {}
+modulated_dict = {}
+
+for key in metaicl_keys:
+    task_name = key.replace("test_metaicl_macro_f1_", "")
+    metaicl_dict[task_name] = all_results[key]
+
+for key in no_context_keys:
+    task_name = key.replace("test_metaicl_no_context_macro_f1_", "")
+    no_context_dict[task_name] = all_results[key]
+
+# Get modulated results (assuming they use the same key pattern as with context)
+for key in all_results_modulated.keys():
+    if key.startswith("test_metaicl_macro_f1_"):
+        task_name = key.replace("test_metaicl_macro_f1_", "")
+        modulated_dict[task_name] = all_results_modulated[key]
+
+# Get all unique task names from the base results (this determines ordering)
+all_tasks = set(metaicl_dict.keys()) | set(no_context_dict.keys())
+
+# Create list of tasks with their differences for sorting (based on without context vs with context)
+task_data = []
+for task in all_tasks:
+    with_context = metaicl_dict.get(task, "N/A")
+    without_context = no_context_dict.get(task, "N/A")
+    modulated = modulated_dict.get(task, "N/A")
+
+    if isinstance(with_context, (int, float)) and isinstance(
+        without_context, (int, float)
+    ):
+        diff_with_context = with_context - without_context
+        modulated_diff = (
+            modulated - without_context
+            if isinstance(modulated, (int, float))
+            else float("-inf")
+        )
+        task_data.append(
+            (
+                task,
+                without_context,
+                with_context,
+                modulated,
+                diff_with_context,
+                modulated_diff,
+            )
+        )
+    else:
+        # Put tasks with missing data at the end
+        task_data.append(
+            (
+                task,
+                without_context,
+                with_context,
+                modulated,
+                float("-inf"),
+                float("-inf"),
+            )
+        )
+
+# Sort by difference (with context - without context) from high to low
+task_data.sort(key=lambda x: x[4], reverse=True)
+
+for (
+    task,
+    without_context,
+    with_context,
+    modulated,
+    diff_with_context,
+    modulated_diff,
+) in task_data:
+    if isinstance(with_context, (int, float)) and isinstance(
+        without_context, (int, float)
+    ):
+        modulated_str = (
+            f"{modulated:.4f}"
+            if isinstance(modulated, (int, float))
+            else str(modulated)
+        )
+        modulated_diff_str = (
+            f"(+{modulated_diff:+.4f})"
+            if isinstance(modulated_diff, (int, float))
+            and modulated_diff != float("-inf")
+            else ""
+        )
+        print(
+            f"{task}: {without_context:.4f} {task}: {with_context:.4f} (+{diff_with_context:+.4f}) {task}: {modulated_str} {modulated_diff_str}"
+        )
+    else:
+        print(f"{task}: {without_context} {task}: {with_context} {task}: {modulated}")
+
+IMPROVEMENT_THRESHOLD = 0.10
+# Filter tasks where with_context shows >5% increase over without_context
+high_improvement_tasks = []
+for (
+    task,
+    without_context,
+    with_context,
+    modulated,
+    diff_with_context,
+    modulated_diff,
+) in task_data:
+    if (
+        isinstance(with_context, (int, float))
+        and isinstance(without_context, (int, float))
+        and isinstance(diff_with_context, (int, float))
+        and diff_with_context > IMPROVEMENT_THRESHOLD
+    ):
+        high_improvement_tasks.append(
+            (
+                task,
+                without_context,
+                with_context,
+                modulated,
+                diff_with_context,
+                modulated_diff,
+            )
+        )
+
+if high_improvement_tasks:
+    print("\n" + "=" * 80)
+    print(
+        f"Tasks with >{IMPROVEMENT_THRESHOLD * 100}% improvement when adding context ({len(high_improvement_tasks)} tasks):"
+    )
+    print("=" * 80)
+
+    # Calculate averages
+    with_context_improvements = [diff for _, _, _, _, diff, _ in high_improvement_tasks]
+    modulated_improvements = [
+        mod_diff
+        for _, _, _, _, _, mod_diff in high_improvement_tasks
+        if isinstance(mod_diff, (int, float)) and mod_diff != float("-inf")
+    ]
+
+    avg_with_context_improvement = sum(with_context_improvements) / len(
+        with_context_improvements
+    )
+    avg_modulated_improvement = (
+        sum(modulated_improvements) / len(modulated_improvements)
+        if modulated_improvements
+        else 0
+    )
+
+    print(f"Average improvement with context: +{avg_with_context_improvement:.4f}")
+    print(f"Average improvement with modulated: +{avg_modulated_improvement:.4f}")
+
+    print("\nDetailed breakdown:")
+    for (
+        task,
+        without_context,
+        with_context,
+        modulated,
+        diff_with_context,
+        modulated_diff,
+    ) in high_improvement_tasks:
+        modulated_str = (
+            f"{modulated:.4f}"
+            if isinstance(modulated, (int, float))
+            else str(modulated)
+        )
+        modulated_diff_str = (
+            f"(+{modulated_diff:+.4f})"
+            if isinstance(modulated_diff, (int, float))
+            and modulated_diff != float("-inf")
+            else ""
+        )
+        print(
+            f"  {task}: {without_context:.4f} → {with_context:.4f} (+{diff_with_context:+.4f}) | Modulated: {modulated_str} {modulated_diff_str}"
+        )
+else:
+    print(
+        f"\nNo tasks found with >{IMPROVEMENT_THRESHOLD * 100}% improvement when adding context."
+    )
diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png
new file mode 100644
index 0000000..80fa024
Binary files /dev/null and b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png differ
diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png
new file mode 100644
index 0000000..700dca4
Binary files /dev/null and b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png differ
diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png
new file mode 100644
index 0000000..5151e90
Binary files /dev/null and b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png differ
diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png
new file mode 100644
index 0000000..0d4aec8
Binary files /dev/null and b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png differ
diff --git a/tmp/packing_debug.log/debug.log b/tmp/packing_debug.log/debug.log
new file mode 100644
index 0000000..a21ceaa
--- /dev/null
+++ b/tmp/packing_debug.log/debug.log
@@ -0,0 +1,2997 @@
+2025-07-03 10:08:32 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 10:08:32 INFO: Starting packing script...
+2025-07-03 10:08:32 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 10:08:32 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 10:08:32 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 10:08:33 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 10:08:34 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 10:08:34 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 10:08:34 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:08:34 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}GemmaTokenizerFast(name_or_path='google/gemma-2-2b-it', vocab_size=256000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='left', special_tokens={'bos_token': '<bos>', 'eos_token': '<eos>', 'unk_token': '<unk>', 'pad_token': '<pad>', 'additional_special_tokens': ['<start_of_turn>', '<end_of_turn>']}, clean_up_tokenization_spaces=False, added_tokens_decoder={
+	0: AddedToken("<pad>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	1: AddedToken("<eos>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	2: AddedToken("<bos>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	3: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	4: AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	5: AddedToken("<2mass>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	6: AddedToken("[@BOS@]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	7: AddedToken("<unused0>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	8: AddedToken("<unused1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	9: AddedToken("<unused2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	10: AddedToken("<unused3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	11: AddedToken("<unused4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	12: AddedToken("<unused5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	13: AddedToken("<unused6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	14: AddedToken("<unused7>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	15: AddedToken("<unused8>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	16: AddedToken("<unused9>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	17: AddedToken("<unused10>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	18: AddedToken("<unused11>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	19: AddedToken("<unused12>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	20: AddedToken("<unused13>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	21: AddedToken("<unused14>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	22: AddedToken("<unused15>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	23: AddedToken("<unused16>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	24: AddedToken("<unused17>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	25: AddedToken("<unused18>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	26: AddedToken("<unused19>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	27: AddedToken("<unused20>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	28: AddedToken("<unused21>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	29: AddedToken("<unused22>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	30: AddedToken("<unused23>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	31: AddedToken("<unused24>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	32: AddedToken("<unused25>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	33: AddedToken("<unused26>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	34: AddedToken("<unused27>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	35: AddedToken("<unused28>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	36: AddedToken("<unused29>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	37: AddedToken("<unused30>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	38: AddedToken("<unused31>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	39: AddedToken("<unused32>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	40: AddedToken("<unused33>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	41: AddedToken("<unused34>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	42: AddedToken("<unused35>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	43: AddedToken("<unused36>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	44: AddedToken("<unused37>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	45: AddedToken("<unused38>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	46: AddedToken("<unused39>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	47: AddedToken("<unused40>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	48: AddedToken("<unused41>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	49: AddedToken("<unused42>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	50: AddedToken("<unused43>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	51: AddedToken("<unused44>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	52: AddedToken("<unused45>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	53: AddedToken("<unused46>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	54: AddedToken("<unused47>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	55: AddedToken("<unused48>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	56: AddedToken("<unused49>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	57: AddedToken("<unused50>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	58: AddedToken("<unused51>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	59: AddedToken("<unused52>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	60: AddedToken("<unused53>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	61: AddedToken("<unused54>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	62: AddedToken("<unused55>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	63: AddedToken("<unused56>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	64: AddedToken("<unused57>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	65: AddedToken("<unused58>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	66: AddedToken("<unused59>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	67: AddedToken("<unused60>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	68: AddedToken("<unused61>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	69: AddedToken("<unused62>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	70: AddedToken("<unused63>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	71: AddedToken("<unused64>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	72: AddedToken("<unused65>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	73: AddedToken("<unused66>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	74: AddedToken("<unused67>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	75: AddedToken("<unused68>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	76: AddedToken("<unused69>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	77: AddedToken("<unused70>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	78: AddedToken("<unused71>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	79: AddedToken("<unused72>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	80: AddedToken("<unused73>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	81: AddedToken("<unused74>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	82: AddedToken("<unused75>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	83: AddedToken("<unused76>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	84: AddedToken("<unused77>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	85: AddedToken("<unused78>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	86: AddedToken("<unused79>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	87: AddedToken("<unused80>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	88: AddedToken("<unused81>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	89: AddedToken("<unused82>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	90: AddedToken("<unused83>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	91: AddedToken("<unused84>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	92: AddedToken("<unused85>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	93: AddedToken("<unused86>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	94: AddedToken("<unused87>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	95: AddedToken("<unused88>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	96: AddedToken("<unused89>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	97: AddedToken("<unused90>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	98: AddedToken("<unused91>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	99: AddedToken("<unused92>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	100: AddedToken("<unused93>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	101: AddedToken("<unused94>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	102: AddedToken("<unused95>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	103: AddedToken("<unused96>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	104: AddedToken("<unused97>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	105: AddedToken("<unused98>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	106: AddedToken("<start_of_turn>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	107: AddedToken("<end_of_turn>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	108: AddedToken("
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	109: AddedToken("
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	110: AddedToken("
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	111: AddedToken("
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	112: AddedToken("
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	113: AddedToken("
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	114: AddedToken("
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	115: AddedToken("
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	116: AddedToken("
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	117: AddedToken("
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	118: AddedToken("
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	119: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	120: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	121: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	122: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	123: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	124: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	125: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	126: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	127: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	128: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	129: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	130: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	131: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	132: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	133: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	134: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	135: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	136: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	137: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	138: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	139: AddedToken("▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	140: AddedToken("▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	141: AddedToken("▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	142: AddedToken("▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	143: AddedToken("▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	144: AddedToken("▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	145: AddedToken("▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	146: AddedToken("▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	147: AddedToken("▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	148: AddedToken("▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	149: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	150: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	151: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	152: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	153: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	154: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	155: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	156: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	157: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	158: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	159: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	160: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	161: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	162: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	163: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	164: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	165: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	166: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	167: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	168: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	169: AddedToken("<table>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	170: AddedToken("<caption>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	171: AddedToken("<thead>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	172: AddedToken("<tbody>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	173: AddedToken("<tfoot>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	174: AddedToken("<tr>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	175: AddedToken("<th>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	176: AddedToken("<td>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	177: AddedToken("</table>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	178: AddedToken("</caption>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	179: AddedToken("</thead>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	180: AddedToken("</tbody>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	181: AddedToken("</tfoot>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	182: AddedToken("</tr>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	183: AddedToken("</th>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	184: AddedToken("</td>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	185: AddedToken("<h1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	186: AddedToken("<h2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	187: AddedToken("<h3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	188: AddedToken("<h4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	189: AddedToken("<h5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	190: AddedToken("<h6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	191: AddedToken("<blockquote>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	192: AddedToken("</h1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	193: AddedToken("</h2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	194: AddedToken("</h3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	195: AddedToken("</h4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	196: AddedToken("</h5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	197: AddedToken("</h6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	198: AddedToken("</blockquote>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	199: AddedToken("<strong>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	200: AddedToken("<em>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	201: AddedToken("<b>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	202: AddedToken("<i>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	203: AddedToken("<u>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	204: AddedToken("<s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	205: AddedToken("<sub>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	206: AddedToken("<sup>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	207: AddedToken("<code>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	208: AddedToken("</strong>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	209: AddedToken("</em>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	210: AddedToken("</b>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	211: AddedToken("</i>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	212: AddedToken("</u>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	213: AddedToken("</s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	214: AddedToken("</sub>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	215: AddedToken("</sup>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	216: AddedToken("</code>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255968: AddedToken("[toxicity=0]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255969: AddedToken("		", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255970: AddedToken("			", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255971: AddedToken("				", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255972: AddedToken("					", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255973: AddedToken("						", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255974: AddedToken("							", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255975: AddedToken("								", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255976: AddedToken("									", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255977: AddedToken("										", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255978: AddedToken("											", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255979: AddedToken("												", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255980: AddedToken("													", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255981: AddedToken("														", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255982: AddedToken("															", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255983: AddedToken("																", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255984: AddedToken("																	", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255985: AddedToken("																		", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255986: AddedToken("																			", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255987: AddedToken("																				", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255988: AddedToken("																					", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255989: AddedToken("																						", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255990: AddedToken("																							", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255991: AddedToken("																								", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255992: AddedToken("																									", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255993: AddedToken("																										", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255994: AddedToken("																											", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255995: AddedToken("																												", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255996: AddedToken("																													", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255997: AddedToken("																														", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255998: AddedToken("																															", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255999: AddedToken("<unused99>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+}
+)GemmaTokenizerFast(name_or_path='google/gemma-2-2b-it', vocab_size=256000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='left', special_tokens={'bos_token': '<bos>', 'eos_token': '<eos>', 'unk_token': '<unk>', 'pad_token': '<pad>', 'additional_special_tokens': ['<start_of_turn>', '<end_of_turn>']}, clean_up_tokenization_spaces=False, added_tokens_decoder={
+	0: AddedToken("<pad>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	1: AddedToken("<eos>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	2: AddedToken("<bos>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	3: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	4: AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	5: AddedToken("<2mass>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	6: AddedToken("[@BOS@]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	7: AddedToken("<unused0>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	8: AddedToken("<unused1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	9: AddedToken("<unused2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	10: AddedToken("<unused3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	11: AddedToken("<unused4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	12: AddedToken("<unused5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	13: AddedToken("<unused6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	14: AddedToken("<unused7>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	15: AddedToken("<unused8>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	16: AddedToken("<unused9>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	17: AddedToken("<unused10>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	18: AddedToken("<unused11>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	19: AddedToken("<unused12>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	20: AddedToken("<unused13>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	21: AddedToken("<unused14>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	22: AddedToken("<unused15>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	23: AddedToken("<unused16>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	24: AddedToken("<unused17>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	25: AddedToken("<unused18>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	26: AddedToken("<unused19>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	27: AddedToken("<unused20>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	28: AddedToken("<unused21>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	29: AddedToken("<unused22>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	30: AddedToken("<unused23>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	31: AddedToken("<unused24>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	32: AddedToken("<unused25>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	33: AddedToken("<unused26>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	34: AddedToken("<unused27>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	35: AddedToken("<unused28>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	36: AddedToken("<unused29>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	37: AddedToken("<unused30>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	38: AddedToken("<unused31>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	39: AddedToken("<unused32>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	40: AddedToken("<unused33>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	41: AddedToken("<unused34>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	42: AddedToken("<unused35>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	43: AddedToken("<unused36>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	44: AddedToken("<unused37>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	45: AddedToken("<unused38>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	46: AddedToken("<unused39>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	47: AddedToken("<unused40>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	48: AddedToken("<unused41>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	49: AddedToken("<unused42>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	50: AddedToken("<unused43>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	51: AddedToken("<unused44>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	52: AddedToken("<unused45>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	53: AddedToken("<unused46>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	54: AddedToken("<unused47>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	55: AddedToken("<unused48>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	56: AddedToken("<unused49>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	57: AddedToken("<unused50>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	58: AddedToken("<unused51>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	59: AddedToken("<unused52>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	60: AddedToken("<unused53>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	61: AddedToken("<unused54>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	62: AddedToken("<unused55>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	63: AddedToken("<unused56>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	64: AddedToken("<unused57>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	65: AddedToken("<unused58>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	66: AddedToken("<unused59>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	67: AddedToken("<unused60>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	68: AddedToken("<unused61>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	69: AddedToken("<unused62>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	70: AddedToken("<unused63>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	71: AddedToken("<unused64>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	72: AddedToken("<unused65>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	73: AddedToken("<unused66>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	74: AddedToken("<unused67>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	75: AddedToken("<unused68>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	76: AddedToken("<unused69>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	77: AddedToken("<unused70>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	78: AddedToken("<unused71>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	79: AddedToken("<unused72>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	80: AddedToken("<unused73>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	81: AddedToken("<unused74>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	82: AddedToken("<unused75>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	83: AddedToken("<unused76>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	84: AddedToken("<unused77>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	85: AddedToken("<unused78>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	86: AddedToken("<unused79>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	87: AddedToken("<unused80>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	88: AddedToken("<unused81>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	89: AddedToken("<unused82>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	90: AddedToken("<unused83>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	91: AddedToken("<unused84>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	92: AddedToken("<unused85>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	93: AddedToken("<unused86>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	94: AddedToken("<unused87>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	95: AddedToken("<unused88>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	96: AddedToken("<unused89>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	97: AddedToken("<unused90>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	98: AddedToken("<unused91>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	99: AddedToken("<unused92>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	100: AddedToken("<unused93>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	101: AddedToken("<unused94>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	102: AddedToken("<unused95>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	103: AddedToken("<unused96>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	104: AddedToken("<unused97>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	105: AddedToken("<unused98>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	106: AddedToken("<start_of_turn>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	107: AddedToken("<end_of_turn>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	108: AddedToken("
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	109: AddedToken("
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	110: AddedToken("
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	111: AddedToken("
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	112: AddedToken("
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	113: AddedToken("
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	114: AddedToken("
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	115: AddedToken("
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	116: AddedToken("
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	117: AddedToken("
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	118: AddedToken("
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	119: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	120: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	121: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	122: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	123: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	124: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	125: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	126: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	127: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	128: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	129: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	130: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	131: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	132: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	133: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	134: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	135: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	136: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	137: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	138: AddedToken("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	139: AddedToken("▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	140: AddedToken("▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	141: AddedToken("▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	142: AddedToken("▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	143: AddedToken("▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	144: AddedToken("▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	145: AddedToken("▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	146: AddedToken("▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	147: AddedToken("▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	148: AddedToken("▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	149: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	150: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	151: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	152: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	153: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	154: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	155: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	156: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	157: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	158: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	159: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	160: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	161: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	162: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	163: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	164: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	165: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	166: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	167: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	168: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	169: AddedToken("<table>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	170: AddedToken("<caption>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	171: AddedToken("<thead>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	172: AddedToken("<tbody>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	173: AddedToken("<tfoot>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	174: AddedToken("<tr>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	175: AddedToken("<th>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	176: AddedToken("<td>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	177: AddedToken("</table>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	178: AddedToken("</caption>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	179: AddedToken("</thead>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	180: AddedToken("</tbody>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	181: AddedToken("</tfoot>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	182: AddedToken("</tr>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	183: AddedToken("</th>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	184: AddedToken("</td>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	185: AddedToken("<h1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	186: AddedToken("<h2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	187: AddedToken("<h3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	188: AddedToken("<h4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	189: AddedToken("<h5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	190: AddedToken("<h6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	191: AddedToken("<blockquote>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	192: AddedToken("</h1>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	193: AddedToken("</h2>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	194: AddedToken("</h3>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	195: AddedToken("</h4>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	196: AddedToken("</h5>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	197: AddedToken("</h6>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	198: AddedToken("</blockquote>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	199: AddedToken("<strong>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	200: AddedToken("<em>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	201: AddedToken("<b>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	202: AddedToken("<i>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	203: AddedToken("<u>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	204: AddedToken("<s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	205: AddedToken("<sub>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	206: AddedToken("<sup>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	207: AddedToken("<code>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	208: AddedToken("</strong>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	209: AddedToken("</em>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	210: AddedToken("</b>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	211: AddedToken("</i>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	212: AddedToken("</u>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	213: AddedToken("</s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	214: AddedToken("</sub>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	215: AddedToken("</sup>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	216: AddedToken("</code>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255968: AddedToken("[toxicity=0]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255969: AddedToken("		", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255970: AddedToken("			", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255971: AddedToken("				", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255972: AddedToken("					", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255973: AddedToken("						", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255974: AddedToken("							", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255975: AddedToken("								", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255976: AddedToken("									", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255977: AddedToken("										", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255978: AddedToken("											", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255979: AddedToken("												", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255980: AddedToken("													", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255981: AddedToken("														", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255982: AddedToken("															", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255983: AddedToken("																", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255984: AddedToken("																	", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255985: AddedToken("																		", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255986: AddedToken("																			", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255987: AddedToken("																				", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255988: AddedToken("																					", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255989: AddedToken("																						", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255990: AddedToken("																							", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255991: AddedToken("																								", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255992: AddedToken("																									", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255993: AddedToken("																										", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255994: AddedToken("																											", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255995: AddedToken("																												", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255996: AddedToken("																													", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255997: AddedToken("																														", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255998: AddedToken("																															", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+	255999: AddedToken("<unused99>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
+}
+)
+2025-07-03 10:08:34 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:08:34 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 10:08:35 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 10:08:35 DEBUG: Attempting to acquire lock 22432746895584 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:08:35 DEBUG: Lock 22432746895584 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:08:35 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:08:35 DEBUG: Attempting to release lock 22432746895584 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:08:35 DEBUG: Lock 22432746895584 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:08:35 DEBUG: Attempting to acquire lock 22432746892656 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:08:35 DEBUG: Lock 22432746892656 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:08:35 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:08:35 DEBUG: Attempting to release lock 22432746892656 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:08:35 DEBUG: Lock 22432746892656 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpzoas0zl6
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpfinusham
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmplwzimi68
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp5cwifvch
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpru97pwia
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpm_5lsjk6
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmptiivg2lg
+2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpcqzm9x1g
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpho0ay9es
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpnquy7ttc
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpqaf0qq0e
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpipy0xquz
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp_godjfvm
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp2fyweh9f
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpgu73efxs
+2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpgj89zh35
+2025-07-03 10:08:45 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp94fjzt08
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmprs9180is
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpzom3vgt5
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmptz_xedub
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpbpow2msk
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp8ytrp5an
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpt5lyfg6j
+2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpq2ak9_ha
+2025-07-03 10:09:21 INFO: Tokenizing 1499551 messages...
+2025-07-03 10:12:14 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp_f82tqbc
+2025-07-03 10:13:08 DEBUG: Attempting to acquire lock 22446846621872 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Lock 22446846621872 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Attempting to release lock 22446846621872 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Lock 22446846621872 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Attempting to acquire lock 22446846623120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Lock 22446846623120 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Attempting to release lock 22446846623120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:13:08 DEBUG: Lock 22446846623120 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:14:27 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 10:14:27 INFO: Starting packing script...
+2025-07-03 10:14:27 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 10:14:27 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 10:14:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 10:14:29 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 10:14:29 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 10:14:29 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 10:14:29 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:14:29 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 10:14:29 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 10:14:29 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:14:29 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 10:14:30 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 10:14:30 DEBUG: Attempting to acquire lock 22478710121344 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:14:30 DEBUG: Lock 22478710121344 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:14:30 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:14:30 DEBUG: Attempting to release lock 22478710121344 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:14:30 DEBUG: Lock 22478710121344 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:14:30 DEBUG: Attempting to acquire lock 22478710120240 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:14:30 DEBUG: Lock 22478710120240 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:14:30 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:14:30 DEBUG: Attempting to release lock 22478710120240 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:14:30 DEBUG: Lock 22478710120240 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpctv11mcw
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpequ_8z7h
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp4tckogmt
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpofu2jvxr
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpzc5pm5p4
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp0mwsm8d2
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp10hz0psh
+2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpo2ry1ela
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpgmq_evz7
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpkkc97a11
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpwk8wx3qr
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpalg7n6db
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp6uze3tom
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp93zj2oyq
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpm3qxwbpb
+2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpygk3d8kc
+2025-07-03 10:14:40 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmprzy2uuh6
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp6i70tn2k
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpsc14yk28
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpoze7h04q
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpl58x9ij3
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpc3u4cro_
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp95wrned0
+2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpb2pmdl4j
+2025-07-03 10:15:16 INFO: Tokenizing 1499551 messages...
+2025-07-03 10:18:05 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpbxaduedp
+2025-07-03 10:20:16 INFO: Tokenizing 1499355 messages...
+2025-07-03 10:24:37 INFO: Tokenizing 641765 messages...
+2025-07-03 10:26:32 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpnaork6vj
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpx9lka_d0
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpycqswfp0
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpxq16nflq
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpsg9fv_g5
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpdkfzk1tz
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpmr27xqzq
+2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpcp6di0wa
+2025-07-03 10:29:04 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp2mbztoy9
+2025-07-03 10:29:21 DEBUG: Attempting to acquire lock 22492809575296 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Lock 22492809575296 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Attempting to release lock 22492809575296 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Lock 22492809575296 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Attempting to acquire lock 22492809574576 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Lock 22492809574576 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Attempting to release lock 22492809574576 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:29:21 DEBUG: Lock 22492809574576 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 10:50:36 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 10:50:36 INFO: Starting packing script...
+2025-07-03 10:50:36 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 10:50:36 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 10:50:36 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 10:50:38 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 10:50:38 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 10:50:38 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 10:50:38 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:50:38 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 10:50:38 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 10:50:38 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 10:50:38 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 10:50:39 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 10:50:39 DEBUG: Attempting to acquire lock 22684547950832 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:50:39 DEBUG: Lock 22684547950832 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:50:39 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:50:39 DEBUG: Attempting to release lock 22684547950832 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:50:39 DEBUG: Lock 22684547950832 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 10:50:39 DEBUG: Attempting to acquire lock 22684547947280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:50:39 DEBUG: Lock 22684547947280 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:50:39 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 10:50:39 DEBUG: Attempting to release lock 22684547947280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:50:39 DEBUG: Lock 22684547947280 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpmv4ehy89
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmppkqcnfcq
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp3wgvbrlq
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpl3c1jz16
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmps63_85v8
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbe55sax2
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpxlh1mbir
+2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpxdp9hkaa
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbg2o_p5d
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpe7w6ky_9
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpd1n7mexm
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpttuwa6tc
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpleql5td5
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmppmix2r6k
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpuvvw86dr
+2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp3qhazjh5
+2025-07-03 10:50:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp5c7umr8l
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp9hxunkv3
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpuow8wxzz
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpb_fbepjc
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpe41pr1lu
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpzyo5rla9
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpzvlv57r7
+2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbutjlxxl
+2025-07-03 10:51:25 INFO: Tokenizing 1499551 messages...
+2025-07-03 10:54:24 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpqyh4h5n4
+2025-07-03 10:56:36 INFO: Tokenizing 1499355 messages...
+2025-07-03 11:00:51 INFO: Tokenizing 641765 messages...
+2025-07-03 11:02:37 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpn07zwasm
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpik5enaxv
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpr877c841
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmptw93t2ov
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp1ploazhg
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp4to_8gh6
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpowqshns2
+2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpd3nlvnuf
+2025-07-03 11:05:08 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp_7ttid7k
+2025-07-03 11:05:19 DEBUG: Attempting to acquire lock 22698589453184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:05:19 DEBUG: Lock 22698589453184 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:05:19 DEBUG: Attempting to release lock 22698589453184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:05:19 DEBUG: Lock 22698589453184 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:05:19 DEBUG: Attempting to acquire lock 22698589452464 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:05:20 DEBUG: Lock 22698589452464 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:05:20 DEBUG: Attempting to release lock 22698589452464 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:05:20 DEBUG: Lock 22698589452464 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:10:10 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:10:10 INFO: Starting packing script...
+2025-07-03 11:10:10 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:10:10 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:10:11 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:10:12 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:10:12 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:10:13 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:10:13 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:10:13 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:10:13 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59
+2025-07-03 11:10:13 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:10:13 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:10:13 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:10:13 DEBUG: Attempting to acquire lock 22683137782880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:10:13 DEBUG: Lock 22683137782880 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:10:13 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:10:13 DEBUG: Attempting to release lock 22683137782880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:10:13 DEBUG: Lock 22683137782880 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:10:13 DEBUG: Attempting to acquire lock 22683137784896 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:10:13 DEBUG: Lock 22683137784896 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:10:13 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:10:13 DEBUG: Attempting to release lock 22683137784896 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:10:13 DEBUG: Lock 22683137784896 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmplqhp3tdc
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpjd70q98h
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmprctpi541
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp6pngkjtu
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp6759ubna
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7n6ktlxp
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpa2zcwvd_
+2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpqunuzogk
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmphbj_y95p
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxy7oy1yn
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7_ibg2z3
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvtwgcngf
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpywxi57z6
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiv59p0ag
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmptuaq784j
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpfi9_nrmk
+2025-07-03 11:10:14 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split...
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpfjlodwq0
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp9h8owzhz
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxc586awi
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvckodjj3
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpwrspo0ex
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxk994zcr
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpi1lw4894
+2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpce7t2fde
+2025-07-03 11:10:17 INFO: Tokenizing 149943 messages...
+2025-07-03 11:10:33 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp_4l3tj0b
+2025-07-03 11:10:46 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp88m1h336
+2025-07-03 11:10:48 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp9h2cb1fu
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpot41f_f2
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp39vb7o54
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7a9xbrb_
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpx5q40t11
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpmxsmtjcs
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpkaqkvel6
+2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmprnaz0x52
+2025-07-03 11:10:58 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpck4im9p9
+2025-07-03 11:11:28 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmptkc2cdu3
+2025-07-03 11:11:45 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiq5wwgkp
+2025-07-03 11:12:22 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp5ae0eci5
+2025-07-03 11:12:47 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpztb00fq4
+2025-07-03 11:12:47 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18742
+# Packed samples: 323
+Avg inp packing efficiency: 0.869
+Avg ctx packing efficiency: 0.978
+
+Input IDs length stats:
+
+  Avg: 7120.9, Std: 769.4, Min: 4961, Max: 8191
+Context IDs length stats:
+
+  Avg: 16027.0, Std: 590.6, Min: 12720, Max: 16384
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.864
+Avg ctx packing efficiency: 0.980
+
+Input IDs length stats:
+
+  Avg: 7077.8, Std: 834.6, Min: 1476, Max: 8188
+Context IDs length stats:
+
+  Avg: 16055.7, Std: 856.4, Min: 3077, Max: 16380
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.869
+Avg ctx packing efficiency: 0.979
+
+Input IDs length stats:
+
+  Avg: 7122.3, Std: 794.3, Min: 710, Max: 8192
+Context IDs length stats:
+
+  Avg: 16037.5, Std: 938.3, Min: 1130, Max: 16384
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.857
+Avg ctx packing efficiency: 0.984
+
+Input IDs length stats:
+
+  Avg: 7022.5, Std: 759.3, Min: 3561, Max: 8187
+Context IDs length stats:
+
+  Avg: 16118.7, Std: 570.1, Min: 8611, Max: 16384
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.866
+Avg ctx packing efficiency: 0.981
+
+Input IDs length stats:
+
+  Avg: 7095.2, Std: 770.7, Min: 4684, Max: 8190
+Context IDs length stats:
+
+  Avg: 16064.8, Std: 580.0, Min: 10728, Max: 16384
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.869
+Avg ctx packing efficiency: 0.980
+
+Input IDs length stats:
+
+  Avg: 7116.9, Std: 800.4, Min: 1397, Max: 8189
+Context IDs length stats:
+
+  Avg: 16059.5, Std: 815.3, Min: 4429, Max: 16384
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.862
+Avg ctx packing efficiency: 0.983
+
+Input IDs length stats:
+
+  Avg: 7058.2, Std: 772.2, Min: 4721, Max: 8192
+Context IDs length stats:
+
+  Avg: 16097.6, Std: 486.8, Min: 12416, Max: 16384
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpotlzqo0m
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvjxl4xqo
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpklqpwy05
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp0w7rdusn
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiit8bd4n
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpziwhqw2d
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpcfiekslg
+2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.865
+Avg ctx packing efficiency: 0.979
+
+Input IDs length stats:
+
+  Avg: 7084.8, Std: 832.0, Min: 960, Max: 8188
+Context IDs length stats:
+
+  Avg: 16044.9, Std: 914.8, Min: 2392, Max: 16384
+2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpj_i6ra_3
+2025-07-03 11:12:58 DEBUG: Attempting to acquire lock 22697169681280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Lock 22697169681280 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Attempting to release lock 22697169681280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Lock 22697169681280 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Attempting to acquire lock 22697169680560 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Lock 22697169680560 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Attempting to release lock 22697169680560 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:12:58 DEBUG: Lock 22697169680560 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:18:05 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:18:05 INFO: Starting packing script...
+2025-07-03 11:18:05 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:18:05 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:18:05 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:18:06 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:18:07 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:18:07 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:18:07 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:18:07 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:18:07 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59
+2025-07-03 11:18:07 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:18:07 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:18:08 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:18:08 DEBUG: Attempting to acquire lock 22346090740848 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:18:08 DEBUG: Lock 22346090740848 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:18:08 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:18:08 DEBUG: Attempting to release lock 22346090740848 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:18:08 DEBUG: Lock 22346090740848 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:18:08 DEBUG: Attempting to acquire lock 22346090735280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:18:08 DEBUG: Lock 22346090735280 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:18:08 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:18:08 DEBUG: Attempting to release lock 22346090735280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:18:08 DEBUG: Lock 22346090735280 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8g5era_n
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp714klyhm
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpf9voq0kk
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp062oeaex
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8d_378dh
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpnkufhl8b
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpp6s1ma4r
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpawz_qg_b
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmppih93v5e
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp0exum433
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpr6hwzfol
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpric2i8_v
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpbo8_439a
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjipxs939
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp7p_bvpmo
+2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp2bi1y45r
+2025-07-03 11:18:08 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split...
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpo0utbnxr
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpuc8v_avj
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp4gbd5kzl
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp_ismds3o
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp9ucr287i
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjuzjhbnr
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpj8h9ggpw
+2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfyxsh1zr
+2025-07-03 11:18:09 DEBUG: Tokenizing inputs
+2025-07-03 11:18:11 INFO: Tokenizing 149943 messages...
+2025-07-03 11:18:28 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp5dbhiws5
+2025-07-03 11:18:39 DEBUG: Tokenizing context
+2025-07-03 11:18:40 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfzoaoh52
+2025-07-03 11:18:43 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmps01yu83_
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpi2xcwdqf
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpivudat_j
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpn3g8rgsz
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmprsx8l7s9
+2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpx8ddazv_
+2025-07-03 11:18:48 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpae9325e6
+2025-07-03 11:18:48 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpu_fy6yaj
+2025-07-03 11:18:52 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjluh649p
+2025-07-03 11:19:23 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmph97o06mk
+2025-07-03 11:19:40 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8kantouc
+2025-07-03 11:20:17 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpxomh3ra5
+2025-07-03 11:20:43 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp42on9wrn
+2025-07-03 11:20:43 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 321
+Avg inp packing efficiency: 0.867
+Avg ctx packing efficiency: 0.984
+
+Input IDs length stats:
+
+  Avg: 7102.6, Std: 727.7, Min: 4866, Max: 8190
+Context IDs length stats:
+
+  Avg: 16115.7, Std: 432.8, Min: 12817, Max: 16383
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 321
+Avg inp packing efficiency: 0.870
+Avg ctx packing efficiency: 0.980
+
+Input IDs length stats:
+
+  Avg: 7127.7, Std: 754.4, Min: 4555, Max: 8186
+Context IDs length stats:
+
+  Avg: 16062.2, Std: 638.8, Min: 8993, Max: 16382
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.864
+Avg ctx packing efficiency: 0.983
+
+Input IDs length stats:
+
+  Avg: 7077.3, Std: 715.4, Min: 5048, Max: 8190
+Context IDs length stats:
+
+  Avg: 16113.1, Std: 448.4, Min: 12101, Max: 16384
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.857
+Avg ctx packing efficiency: 0.978
+
+Input IDs length stats:
+
+  Avg: 7019.1, Std: 827.2, Min: 2735, Max: 8192
+Context IDs length stats:
+
+  Avg: 16030.1, Std: 781.3, Min: 5173, Max: 16383
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.863
+Avg ctx packing efficiency: 0.981
+
+Input IDs length stats:
+
+  Avg: 7065.7, Std: 813.1, Min: 1161, Max: 8191
+Context IDs length stats:
+
+  Avg: 16075.7, Std: 873.4, Min: 2308, Max: 16384
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfp4hgkwb
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 322
+Avg inp packing efficiency: 0.868
+Avg ctx packing efficiency: 0.984
+
+Input IDs length stats:
+
+  Avg: 7106.7, Std: 728.3, Min: 5117, Max: 8188
+Context IDs length stats:
+
+  Avg: 16113.9, Std: 541.2, Min: 10593, Max: 16384
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp9le6uwja
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743
+# Packed samples: 323
+Avg inp packing efficiency: 0.862
+Avg ctx packing efficiency: 0.981
+
+Input IDs length stats:
+
+  Avg: 7058.1, Std: 817.0, Min: 302, Max: 8191
+Context IDs length stats:
+
+  Avg: 16066.2, Std: 894.7, Min: 1761, Max: 16384
+2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18742
+# Packed samples: 323
+Avg inp packing efficiency: 0.877
+Avg ctx packing efficiency: 0.978
+
+Input IDs length stats:
+
+  Avg: 7185.8, Std: 770.3, Min: 3765, Max: 8191
+Context IDs length stats:
+
+  Avg: 16028.5, Std: 710.8, Min: 7969, Max: 16384
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpuwzi6dvu
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpak0dghww
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpnbj_jby2
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfesj1hsi
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpyume5l4e
+2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp3d_k3cud
+2025-07-03 11:20:53 DEBUG: Attempting to acquire lock 22360189877120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Lock 22360189877120 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Attempting to release lock 22360189877120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Lock 22360189877120 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Attempting to acquire lock 22360189876400 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Lock 22360189876400 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Attempting to release lock 22360189876400 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:20:53 DEBUG: Lock 22360189876400 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:23:45 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:23:45 INFO: Starting packing script...
+2025-07-03 11:23:45 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:23:45 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:23:45 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:23:46 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:23:47 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:23:47 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:23:47 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:23:47 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:23:47 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59
+2025-07-03 11:23:47 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:23:47 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:23:48 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:23:48 DEBUG: Attempting to acquire lock 23209466157504 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:23:48 DEBUG: Lock 23209466157504 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:23:48 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:23:48 DEBUG: Attempting to release lock 23209466157504 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:23:48 DEBUG: Lock 23209466157504 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:23:48 DEBUG: Attempting to acquire lock 23209466156592 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:23:48 DEBUG: Lock 23209466156592 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:23:48 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:23:48 DEBUG: Attempting to release lock 23209466156592 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:23:48 DEBUG: Lock 23209466156592 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpfec7vl4l
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpysiy_p01
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpbn_a1b62
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpj6hgwiug
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpgt68oo3e
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpfhbs6tm1
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp90i_36e6
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpsryi9dhz
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptd1ynpn5
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmph4gy6_dd
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpf_cwbp90
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmppkh_biyj
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpt_h8xham
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpeqljf18i
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpibzkd2ix
+2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpo2041sis
+2025-07-03 11:23:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split...
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpgmmgkser
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpk288x_lr
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpuc9v6pu2
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4s34rjkd
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpz6ea_s79
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp1xrgrf6i
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpxlxmzg_y
+2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmphhykjz0s
+2025-07-03 11:23:49 DEBUG: Tokenizing inputs
+2025-07-03 11:23:52 INFO: Tokenizing 149943 messages...
+2025-07-03 11:24:08 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptbba2t_2
+2025-07-03 11:24:18 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpog17x3g4
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4ey34hdk
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpnk_ua7cz
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpxcc6hmci
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp88f6s6fc
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpa8l78q6p
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpl1i3283r
+2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpmmj1jo7a
+2025-07-03 11:24:25 DEBUG: Tokenizing context
+2025-07-03 11:24:27 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptbw3it3a
+2025-07-03 11:24:29 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmphfkyd6sv
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmprnu0y73f
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp0b336u8q
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4sal5bqz
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpber1v2pi
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptejpz18j
+2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpufj1vbf3
+2025-07-03 11:24:35 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp42cex_yj
+2025-07-03 11:24:39 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp0k5sjd0z
+2025-07-03 11:25:10 DEBUG: Attempting to acquire lock 23223565334704 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Lock 23223565334704 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Attempting to release lock 23223565334704 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Lock 23223565334704 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Attempting to acquire lock 23223565335952 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Lock 23223565335952 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Attempting to release lock 23223565335952 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:25:10 DEBUG: Lock 23223565335952 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:25:26 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:25:26 INFO: Starting packing script...
+2025-07-03 11:25:26 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:25:26 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:25:26 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:25:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:25:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:25:28 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:25:28 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:25:28 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:25:28 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59
+2025-07-03 11:25:28 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:25:28 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:25:29 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:25:29 DEBUG: Attempting to acquire lock 22885911361136 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:25:29 DEBUG: Lock 22885911361136 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:25:29 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:25:29 DEBUG: Attempting to release lock 22885911361136 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:25:29 DEBUG: Lock 22885911361136 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:25:29 DEBUG: Attempting to acquire lock 22885911358880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:25:29 DEBUG: Lock 22885911358880 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:25:29 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:25:29 DEBUG: Attempting to release lock 22885911358880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:25:29 DEBUG: Lock 22885911358880 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpy5fwoz50
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpbrdl5yn9
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpe3d_frcd
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxnnn6fep
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp9km1e6nh
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp6pdnoa5u
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmprtvlzxai
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpsqojgc96
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpcxr9l95w
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpa3ldk1_a
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpotvktm_l
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpns59o4cb
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpgd9a03ph
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpr8reidgr
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpas44yp08
+2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpqnqj9q_3
+2025-07-03 11:25:30 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split...
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpugq344fy
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpm5u4x1ya
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxbmgzcpx
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmprf0sohhb
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpp2dqadfv
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmppp99qmln
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3hnry9yr
+2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp_m7w_l1m
+2025-07-03 11:25:30 DEBUG: Tokenizing inputs
+2025-07-03 11:25:32 INFO: Tokenizing 149943 messages...
+2025-07-03 11:25:48 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpq507t7y7
+2025-07-03 11:25:59 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzqsm7c1f
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpywr6xcho
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpldf6bsqr
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmphr9728b1
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp8tk2oetl
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpkydnfjya
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn32ja7yo
+2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp47z6fa1l
+2025-07-03 11:26:06 DEBUG: Tokenizing context
+2025-07-03 11:26:08 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3c8wg3li
+2025-07-03 11:26:10 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp7b92vgqf
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpljenzlqz
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpfw_j_cam
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzjzi4gf2
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn17kcfan
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn0bo6r19
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzb3qin2z
+2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4pesbzlz
+2025-07-03 11:26:15 DEBUG: Unpacking data
+2025-07-03 11:26:15 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmps7fx04bu
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxud5axcz
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp8a4l7orl
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp2nb8htq9
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpt5odwoy5
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4ofpmull
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp674xd6j9
+2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpp_vgnz1a
+2025-07-03 11:26:22 DEBUG: Split too long QAs with max length 2048
+2025-07-03 11:26:22 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 12709
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmplizfckpl
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 7701
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2046
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpquc67_9k
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 7088
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2047
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3zm_5gtg
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 10478
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 8086
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 10726
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpme8uky3f
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 8052
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpr8qwayr3
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmptf17s1gl
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpspuxwo33
+2025-07-03 11:26:27 DEBUG: Longest old qas len: 9002
+2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048
+2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpi4jw6nia
+2025-07-03 11:26:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpru_vctzw
+2025-07-03 11:26:29 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 305
+Avg inp packing efficiency: 0.899
+Avg ctx packing efficiency: 0.091
+
+Input IDs length stats:
+
+  Avg: 7361.2, Std: 520.3, Min: 6151, Max: 8187
+Context IDs length stats:
+
+  Avg: 1490.2, Std: 352.2, Min: 739, Max: 2553
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 306
+Avg inp packing efficiency: 0.901
+Avg ctx packing efficiency: 0.091
+
+Input IDs length stats:
+
+  Avg: 7377.6, Std: 537.4, Min: 5115, Max: 8189
+Context IDs length stats:
+
+  Avg: 1487.4, Std: 366.0, Min: 743, Max: 2707
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 309
+Avg inp packing efficiency: 0.906
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7418.8, Std: 603.6, Min: 1933, Max: 8183
+Context IDs length stats:
+
+  Avg: 1461.0, Std: 365.0, Min: 247, Max: 2605
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxnd8oqqb
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 308
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7401.4, Std: 597.8, Min: 2003, Max: 8192
+Context IDs length stats:
+
+  Avg: 1453.7, Std: 352.5, Min: 361, Max: 2671
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 312
+Avg inp packing efficiency: 0.900
+Avg ctx packing efficiency: 0.088
+
+Input IDs length stats:
+
+  Avg: 7372.1, Std: 593.4, Min: 1845, Max: 8190
+Context IDs length stats:
+
+  Avg: 1437.9, Std: 349.4, Min: 298, Max: 2735
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 305
+Avg inp packing efficiency: 0.904
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7402.1, Std: 578.6, Min: 3633, Max: 8191
+Context IDs length stats:
+
+  Avg: 1465.7, Std: 334.6, Min: 720, Max: 2465
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpmfgpy69q
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3v0a04vn
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 318
+Avg inp packing efficiency: 0.901
+Avg ctx packing efficiency: 0.086
+
+Input IDs length stats:
+
+  Avg: 7378.8, Std: 554.2, Min: 4579, Max: 8191
+Context IDs length stats:
+
+  Avg: 1414.7, Std: 336.9, Min: 737, Max: 2537
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpg8ekl0dk
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpriqbx70d
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4cexfgee
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpt_xgct68
+2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 312
+Avg inp packing efficiency: 0.902
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7392.3, Std: 595.2, Min: 1850, Max: 8192
+Context IDs length stats:
+
+  Avg: 1450.7, Std: 367.5, Min: 150, Max: 3108
+2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpb1qfrx_i
+2025-07-03 11:26:35 DEBUG: Attempting to acquire lock 22900010351488 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Lock 22900010351488 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Attempting to release lock 22900010351488 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Lock 22900010351488 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Attempting to acquire lock 22900010350768 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Lock 22900010350768 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Attempting to release lock 22900010350768 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:26:35 DEBUG: Lock 22900010350768 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:33:14 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:33:14 INFO: Starting packing script...
+2025-07-03 11:33:14 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:33:14 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:33:14 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:33:15 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:33:15 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:33:16 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:33:16 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:33:16 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:33:16 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59
+2025-07-03 11:33:16 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]...
+2025-07-03 11:33:16 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:33:16 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:33:16 DEBUG: Attempting to acquire lock 23413206015936 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:33:16 DEBUG: Lock 23413206015936 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:33:16 DEBUG: Attempting to release lock 23413206015936 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:33:16 DEBUG: Lock 23413206015936 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:33:16 DEBUG: Attempting to acquire lock 23413206015840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:33:16 DEBUG: Lock 23413206015840 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:33:16 DEBUG: Attempting to release lock 23413206015840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:33:16 DEBUG: Lock 23413206015840 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp3iow_g97
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp4acu9dnm
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpw806zxch
+2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpa6sp_k1b
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp8omef5zz
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmptzeftfow
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp99_jyd_3
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_6aop2jd
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpo85hrh2_
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpgwlhkeef
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpnd0lmu6j
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpb7vkkppc
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp7_n4eq37
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmplrlyd652
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpc6btfw7a
+2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpskd9u5p_
+2025-07-03 11:33:18 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split...
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpbaks5ics
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp9la0c5af
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpb_1cx56l
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpv1irw8qw
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2_frssya
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpft84b0js
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpymr73n8_
+2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwwvivbog
+2025-07-03 11:33:18 DEBUG: Tokenizing inputs
+2025-07-03 11:33:22 INFO: Tokenizing 149943 messages...
+2025-07-03 11:33:38 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_2p6th06
+2025-07-03 11:33:50 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:33:50 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmppfao83jd
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp0_9qxggx
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp0d4a_n7j
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6spvygl_
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpuqurh2hq
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpohp_srcq
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmptbmao1iz
+2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpvy2ko277
+2025-07-03 11:33:57 DEBUG: Tokenizing context
+2025-07-03 11:33:58 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwe3e38xz
+2025-07-03 11:34:06 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpm923uoci
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmprjrxv5u9
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpiv4pycn9
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2uf925qv
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2_yi8b47
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6iqs7a5x
+2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpq3cubq4i
+2025-07-03 11:34:11 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp61crqj99
+2025-07-03 11:34:12 DEBUG: Unpacking data
+2025-07-03 11:34:12 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpymuc2336
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpadowp4en
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpl899did7
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp7s5m7k70
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpmm7wmug_
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpzfew9k6i
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp5kt6v9rd
+2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpm94cswgg
+2025-07-03 11:34:18 DEBUG: Split too long QAs with max length 2048
+2025-07-03 11:34:18 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 7088
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2047
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2tqrekv5
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 7701
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2046
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpxr1nve5w
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 8052
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 8086
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpotwjfepd
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpfjk164jy
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 12709
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6a7vbi_3
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 9002
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpuqhczdye
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 10726
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpox70t9za
+2025-07-03 11:34:23 DEBUG: Longest old qas len: 10478
+2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048
+2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpl3chauqt
+2025-07-03 11:34:25 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpur8k2bxl
+2025-07-03 11:34:25 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 312
+Avg inp packing efficiency: 0.895
+Avg ctx packing efficiency: 0.090
+
+Input IDs length stats:
+
+  Avg: 7329.7, Std: 575.8, Min: 1937, Max: 8182
+Context IDs length stats:
+
+  Avg: 1473.5, Std: 391.4, Min: 269, Max: 2746
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 304
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.090
+
+Input IDs length stats:
+
+  Avg: 7399.6, Std: 596.4, Min: 1741, Max: 8187
+Context IDs length stats:
+
+  Avg: 1477.2, Std: 364.4, Min: 77, Max: 2929
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 309
+Avg inp packing efficiency: 0.901
+Avg ctx packing efficiency: 0.090
+
+Input IDs length stats:
+
+  Avg: 7382.4, Std: 495.5, Min: 6182, Max: 8190
+Context IDs length stats:
+
+  Avg: 1473.0, Std: 358.3, Min: 511, Max: 2732
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 306
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.090
+
+Input IDs length stats:
+
+  Avg: 7399.3, Std: 543.3, Min: 5268, Max: 8190
+Context IDs length stats:
+
+  Avg: 1471.0, Std: 352.9, Min: 761, Max: 2731
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 307
+Avg inp packing efficiency: 0.906
+Avg ctx packing efficiency: 0.090
+
+Input IDs length stats:
+
+  Avg: 7424.7, Std: 593.1, Min: 2040, Max: 8189
+Context IDs length stats:
+
+  Avg: 1468.3, Std: 377.4, Min: 657, Max: 2476
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655
+# Packed samples: 313
+Avg inp packing efficiency: 0.905
+Avg ctx packing efficiency: 0.088
+
+Input IDs length stats:
+
+  Avg: 7412.2, Std: 480.0, Min: 6258, Max: 8184
+Context IDs length stats:
+
+  Avg: 1440.5, Std: 333.8, Min: 713, Max: 2496
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_8ng95dx
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpn8vn3gl6
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6b36952_
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 310
+Avg inp packing efficiency: 0.910
+Avg ctx packing efficiency: 0.087
+
+Input IDs length stats:
+
+  Avg: 7452.5, Std: 543.1, Min: 5253, Max: 8192
+Context IDs length stats:
+
+  Avg: 1433.4, Std: 345.5, Min: 697, Max: 2485
+2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654
+# Packed samples: 310
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.088
+
+Input IDs length stats:
+
+  Avg: 7399.8, Std: 632.4, Min: 1274, Max: 8192
+Context IDs length stats:
+
+  Avg: 1441.6, Std: 343.1, Min: 637, Max: 2813
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpddree1qo
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpk7c7dq8t
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpmyru39ll
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpkzz3opiy
+2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwfptlbzo
+2025-07-03 11:34:31 DEBUG: Attempting to acquire lock 23427306306384 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Lock 23427306306384 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Attempting to release lock 23427306306384 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Lock 23427306306384 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Attempting to acquire lock 23427306305664 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Lock 23427306305664 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Attempting to release lock 23427306305664 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:34:31 DEBUG: Lock 23427306305664 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:35:18 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 11:35:18 INFO: Starting packing script...
+2025-07-03 11:35:18 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 11:35:18 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 11:35:18 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 11:35:19 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 11:35:20 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 11:35:20 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 11:35:20 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 11:35:20 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 11:35:20 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 11:35:20 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 11:35:20 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 11:35:21 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 11:35:21 DEBUG: Attempting to acquire lock 22909864697312 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:35:21 DEBUG: Lock 22909864697312 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:35:21 DEBUG: Attempting to release lock 22909864697312 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:35:21 DEBUG: Lock 22909864697312 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 11:35:21 DEBUG: Attempting to acquire lock 22909864701008 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:35:21 DEBUG: Lock 22909864701008 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 11:35:21 DEBUG: Attempting to release lock 22909864701008 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:35:21 DEBUG: Lock 22909864701008 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpg6t5m5rz
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphunzvaq4
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpd1_f9lw2
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpr9vqkc81
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpucggpm9g
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpu_27oix1
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpgtp6vv1q
+2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqzov2ae1
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpy_8_oizg
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp9ww_zy_q
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppypu3w71
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp5eq7w7zo
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp1rira0hc
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmph1_ybwe1
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4zpomupi
+2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphpzj44b6
+2025-07-03 11:35:32 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpyrol6uvi
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpsh9k69if
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpuwd4_eoh
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqkf91a81
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpxt3tjg2i
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpoa8b2pbm
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpj95ctlj6
+2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpb6u_879q
+2025-07-03 11:35:43 DEBUG: Tokenizing inputs
+2025-07-03 11:36:10 INFO: Tokenizing 1499551 messages...
+2025-07-03 11:39:01 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmplkcsgio9
+2025-07-03 11:41:30 INFO: Tokenizing 1499355 messages...
+2025-07-03 11:46:10 INFO: Tokenizing 641765 messages...
+2025-07-03 11:48:06 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpbe2pu0zt
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp604ef9nv
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpnauk7vkk
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphnhqefnt
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpfwassucc
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpmr2kdbwi
+2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpommhop29
+2025-07-03 11:48:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp7rlckzvw
+2025-07-03 11:50:27 DEBUG: Tokenizing context
+2025-07-03 11:50:41 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp7fnpnabo
+2025-07-03 11:53:14 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:53:20 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp53rf3qfz
+2025-07-03 11:53:20 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp__a4ilv6
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmprpdjdszh
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppdi34mys
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpc_p6mol0
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp254gcs6r
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppdzxvk0g
+2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp021gol7o
+2025-07-03 11:55:21 DEBUG: Unpacking data
+2025-07-03 11:55:21 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpw2g7rzn9
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpxgpt4kjo
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpttv63g6r
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpdqq8c4an
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4x6j4ub5
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp13syypyu
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmputn3kwu7
+2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpio252e_2
+2025-07-03 11:57:35 DEBUG: Split too long QAs with max length 2048
+2025-07-03 11:57:35 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 11:59:35 DEBUG: Longest old qas len: 12226
+2025-07-03 11:59:35 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:35 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpej90z4gx
+2025-07-03 11:59:36 DEBUG: Longest old qas len: 12527
+2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpju9zzp11
+2025-07-03 11:59:36 DEBUG: Longest old qas len: 11962
+2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4t1vhbeg
+2025-07-03 11:59:36 DEBUG: Longest old qas len: 11859
+2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp0dx6i6lv
+2025-07-03 11:59:36 DEBUG: Longest old qas len: 12674
+2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqufjvk04
+2025-07-03 11:59:37 DEBUG: Longest old qas len: 12401
+2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp2qr5hr6p
+2025-07-03 11:59:37 DEBUG: Longest old qas len: 12709
+2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphve8947h
+2025-07-03 11:59:37 DEBUG: Longest old qas len: 12850
+2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048
+2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmps_u40ken
+2025-07-03 11:59:40 DEBUG: Attempting to acquire lock 22923963693904 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:59:40 DEBUG: Lock 22923963693904 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:59:40 DEBUG: Attempting to release lock 22923963693904 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:59:40 DEBUG: Lock 22923963693904 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 11:59:40 DEBUG: Attempting to acquire lock 22923963693184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:59:40 DEBUG: Lock 22923963693184 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:59:41 DEBUG: Attempting to release lock 22923963693184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 11:59:41 DEBUG: Lock 22923963693184 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 12:03:42 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 12:03:42 INFO: Starting packing script...
+2025-07-03 12:03:42 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 12:03:42 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 12:03:42 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 12:03:44 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 12:03:44 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 12:03:45 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 12:03:45 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 12:03:45 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 12:03:45 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 12:03:45 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 12:03:45 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 12:03:45 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 12:03:45 DEBUG: Attempting to acquire lock 22872660751840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 12:03:45 DEBUG: Lock 22872660751840 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 12:03:45 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 12:03:45 DEBUG: Attempting to release lock 22872660751840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 12:03:45 DEBUG: Lock 22872660751840 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 12:03:45 DEBUG: Attempting to acquire lock 22872660752560 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 12:03:45 DEBUG: Lock 22872660752560 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 12:03:45 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 12:03:45 DEBUG: Attempting to release lock 22872660752560 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 12:03:45 DEBUG: Lock 22872660752560 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpatxylmqc
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp0k8k8zr1
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfhjxybgj
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp7u13ypfm
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu434v69u
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp8jocb6v0
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp677bi28y
+2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpzu483f9t
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvb3_d79c
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpby_f98i4
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp9md5vllh
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpa2fy0dvk
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp3n5l2h4e
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpkk757vk3
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpr842lqmn
+2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfs2efbik
+2025-07-03 12:03:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwfnsrbca
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2o2tyv3n
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp4u32hk3l
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpre39cklr
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmptox5kjpe
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpznp702x3
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdyhtxob0
+2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp6w6ewv2u
+2025-07-03 12:03:58 DEBUG: Tokenizing inputs
+2025-07-03 12:04:18 INFO: Tokenizing 1499551 messages...
+2025-07-03 12:07:09 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp4n0kmslm
+2025-07-03 12:09:23 INFO: Tokenizing 1499355 messages...
+2025-07-03 12:13:41 INFO: Tokenizing 641765 messages...
+2025-07-03 12:15:30 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpm6vpm_hg
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpr4h3vgk4
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu9m0br2c
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpy5wb3_m8
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu6a91n1l
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2idfwt9i
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp35oyifub
+2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmptc9fiamk
+2025-07-03 12:17:38 DEBUG: Tokenizing context
+2025-07-03 12:17:51 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpyxcnznhz
+2025-07-03 12:18:37 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdph980y3
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpk3n13ft6
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp8drlpk5o
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpa3xpn4ln
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvcx577xh
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpx5ul63xj
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfjf8u9rb
+2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp6t8ga2po
+2025-07-03 12:20:41 DEBUG: Unpacking data
+2025-07-03 12:20:41 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwy16gmju
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwqge6832
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdoa3khov
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp7h5ooggt
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2_ieaazs
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp5h66kvhx
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpnfbqe_wk
+2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpiryfqlj9
+2025-07-03 12:23:17 DEBUG: Split too long QAs with max length 2048
+2025-07-03 12:23:17 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 12:25:30 DEBUG: Longest old qas len: 12401
+2025-07-03 12:25:30 DEBUG: Longest new qas len: 2048
+2025-07-03 12:25:30 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmppodex5po
+2025-07-03 12:25:34 DEBUG: Longest old qas len: 11859
+2025-07-03 12:25:34 DEBUG: Longest new qas len: 2048
+2025-07-03 12:25:34 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp179kperq
+2025-07-03 12:25:34 DEBUG: Longest old qas len: 12226
+2025-07-03 12:25:34 DEBUG: Longest new qas len: 2048
+2025-07-03 12:25:34 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpl0_rkrgf
+2025-07-03 12:25:35 DEBUG: Longest old qas len: 12674
+2025-07-03 12:25:35 DEBUG: Longest new qas len: 2048
+2025-07-03 12:25:35 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvzg39rhn
+2025-07-03 12:25:35 DEBUG: Longest old qas len: 12527
+2025-07-03 12:25:35 DEBUG: Longest new qas len: 2048
+2025-07-03 12:25:35 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpaqqtl3ge
+2025-07-03 12:25:36 DEBUG: Attempting to acquire lock 22886692912000 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Lock 22886692912000 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Attempting to release lock 22886692912000 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Lock 22886692912000 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Attempting to acquire lock 22886692911280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Lock 22886692911280 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Attempting to release lock 22886692911280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 12:25:36 DEBUG: Lock 22886692911280 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:01:20 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 14:01:20 INFO: Starting packing script...
+2025-07-03 14:01:20 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 14:01:20 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 14:01:20 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 14:01:22 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 14:01:22 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 14:01:23 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 14:01:23 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 14:01:23 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 14:01:23 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 14:01:23 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 14:01:23 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 14:01:23 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 14:01:23 DEBUG: Attempting to acquire lock 22684941774000 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:01:23 DEBUG: Lock 22684941774000 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:01:23 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 14:01:23 DEBUG: Attempting to release lock 22684941774000 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:01:23 DEBUG: Lock 22684941774000 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:01:23 DEBUG: Attempting to acquire lock 22684941770448 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:01:23 DEBUG: Lock 22684941770448 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:01:23 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 14:01:23 DEBUG: Attempting to release lock 22684941770448 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:01:23 DEBUG: Lock 22684941770448 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp65chd13a
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmphtvax5kh
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpnoxtu_ht
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp1ibijg8u
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpf4rjzflk
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp7nlj091y
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp438oajg8
+2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpirdlu1ac
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpeg1k3qzr
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpt8j8odur
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjri69jc2
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpejubln25
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpn901kapf
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmposb8b4cp
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjrkcy79f
+2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp699maqif
+2025-07-03 14:01:27 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmph5cpx91l
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpff2pcpuv
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp6x42353v
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpyjxtjuxz
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9i7d7cgm
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcugne3h6
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpckb_sydx
+2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9lofdv2h
+2025-07-03 14:01:36 DEBUG: Tokenizing inputs
+2025-07-03 14:01:56 INFO: Tokenizing 1499551 messages...
+2025-07-03 14:04:46 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpuh_ybp3y
+2025-07-03 14:06:57 INFO: Tokenizing 1499355 messages...
+2025-07-03 14:11:12 INFO: Tokenizing 641765 messages...
+2025-07-03 14:13:00 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmphes6tbze
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpdmq6k39r
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp5garo4er
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpxktzvsiz
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmps_hnfb31
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpx8ilf4qh
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpja0vasr6
+2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpqytlu_m5
+2025-07-03 14:15:05 DEBUG: Tokenizing context
+2025-07-03 14:15:17 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpnjm53jwr
+2025-07-03 14:16:03 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmplgmf33m6
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp_8bsks6_
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpuo5wr0qy
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpyx0fk9el
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpoocxr6if
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpt4pi0u3g
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp8xtuyymz
+2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpg38766pj
+2025-07-03 14:17:59 DEBUG: Unpacking data
+2025-07-03 14:17:59 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcyhhyv31
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjegdtgn6
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp8s4q0ngs
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp0b1ifj7l
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmppc7cu56f
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9m0h47mm
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp57gx_9_5
+2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp3sj9o07x
+2025-07-03 14:20:09 DEBUG: Split too long QAs with max length 2048
+2025-07-03 14:23:24 DEBUG: Longest old qas len: 12850
+2025-07-03 14:23:24 DEBUG: Longest new qas len: 2048
+2025-07-03 14:23:24 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp1bwpue3l
+2025-07-03 14:28:01 DEBUG: Longest old qas len: 12674
+2025-07-03 14:28:01 DEBUG: Longest new qas len: 2048
+2025-07-03 14:30:41 DEBUG: Longest old qas len: 11859
+2025-07-03 14:30:41 DEBUG: Longest new qas len: 2048
+2025-07-03 14:31:16 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcuhk8l19
+2025-07-03 14:31:16 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:32:46 DEBUG: Attempting to acquire lock 22699041521040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Lock 22699041521040 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Attempting to release lock 22699041521040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Lock 22699041521040 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Attempting to acquire lock 22699041520512 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Lock 22699041520512 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Attempting to release lock 22699041520512 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:32:46 DEBUG: Lock 22699041520512 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:42:46 INFO: Logging to: tmp/packing_debug.log/debug.log
+2025-07-03 14:42:46 INFO: Starting packing script...
+2025-07-03 14:42:46 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False}
+2025-07-03 14:42:46 DEBUG: Starting new HTTPS connection (1): huggingface.co:443
+2025-07-03 14:42:46 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0
+2025-07-03 14:42:48 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0
+2025-07-03 14:42:48 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2025-07-03 14:42:49 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja
+2025-07-03 14:42:49 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 14:42:49 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it
+2025-07-03 14:42:49 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3
+2025-07-03 14:42:49 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train...
+2025-07-03 14:42:49 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443
+2025-07-03 14:42:49 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0
+2025-07-03 14:42:49 DEBUG: Attempting to acquire lock 22852114429216 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:42:49 DEBUG: Lock 22852114429216 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:42:49 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 14:42:49 DEBUG: Attempting to release lock 22852114429216 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:42:49 DEBUG: Lock 22852114429216 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock
+2025-07-03 14:42:49 DEBUG: Attempting to acquire lock 22852114423552 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:42:49 DEBUG: Lock 22852114423552 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:42:49 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json
+2025-07-03 14:42:49 DEBUG: Attempting to release lock 22852114423552 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:42:49 DEBUG: Lock 22852114423552 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock
+2025-07-03 14:42:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split...
+2025-07-03 14:42:50 DEBUG: Tokenizing inputs
+2025-07-03 14:42:51 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:42:53 DEBUG: Tokenizing context
+2025-07-03 14:42:54 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes.
+2025-07-03 14:42:54 DEBUG: Unpacking data
+2025-07-03 14:42:55 DEBUG: Split too long QAs with max length 2048
+2025-07-03 14:45:01 DEBUG: Longest old qas len: 12674
+2025-07-03 14:45:01 DEBUG: Longest new qas len: 2048
+2025-07-03 14:45:01 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens)
+2025-07-03 14:45:01 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpdhnf3bxy
+2025-07-03 14:45:02 DEBUG: Longest old qas len: 12226
+2025-07-03 14:45:02 DEBUG: Longest new qas len: 2048
+2025-07-03 14:45:02 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens)
+2025-07-03 14:45:02 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpd8vu0ypo
+2025-07-03 14:45:03 DEBUG: Longest old qas len: 12709
+2025-07-03 14:45:03 DEBUG: Longest new qas len: 2048
+2025-07-03 14:45:03 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens)
+2025-07-03 14:45:03 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmp946mkwxa
+2025-07-03 14:45:04 DEBUG: Longest old qas len: 12850
+2025-07-03 14:45:04 DEBUG: Longest new qas len: 2048
+2025-07-03 14:45:04 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens)
+2025-07-03 14:45:04 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpu6o_ygx9
+2025-07-03 14:45:52 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmplp6me2_j
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00000-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00001-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00002-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00003-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00004-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00005-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00006-of-00015.arrow
+2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00007-of-00015.arrow
+2025-07-03 14:46:31 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00008-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00009-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00010-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00011-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00012-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00014-of-00015.arrow
+2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00013-of-00015.arrow
+2025-07-03 14:46:38 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/state.json
+2025-07-03 14:46:38 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/dataset_info.json
+2025-07-03 14:48:19 DEBUG: Packing stats - Original samples: 80046
+# Packed samples: 14967
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7399.1, Std: 508.8, Min: 4053, Max: 8192
+Context IDs length stats:
+
+  Avg: 1455.9, Std: 346.3, Min: 481, Max: 3053
+2025-07-03 14:48:21 DEBUG: Packing stats - Original samples: 80045
+# Packed samples: 14985
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7398.7, Std: 507.8, Min: 6150, Max: 8192
+Context IDs length stats:
+
+  Avg: 1451.4, Std: 338.9, Min: 563, Max: 3196
+2025-07-03 14:48:21 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpcbf0v49h
+2025-07-03 14:48:21 DEBUG: Packing stats - Original samples: 80046
+# Packed samples: 14984
+Avg inp packing efficiency: 0.903
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7398.8, Std: 511.6, Min: 2849, Max: 8192
+Context IDs length stats:
+
+  Avg: 1454.8, Std: 342.6, Min: 466, Max: 3327
+2025-07-03 14:48:22 DEBUG: Packing stats - Original samples: 80045
+# Packed samples: 14967
+Avg inp packing efficiency: 0.904
+Avg ctx packing efficiency: 0.089
+
+Input IDs length stats:
+
+  Avg: 7403.8, Std: 511.3, Min: 6161, Max: 8192
+Context IDs length stats:
+
+  Avg: 1455.5, Std: 343.7, Min: 510, Max: 3459
+2025-07-03 14:48:23 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpy0ut6956
+2025-07-03 14:48:23 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmp5bcdvqml
+2025-07-03 14:48:24 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpykj2tbpo
+2025-07-03 14:48:36 DEBUG: Attempting to acquire lock 22866146311040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Lock 22866146311040 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Attempting to release lock 22866146311040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Lock 22866146311040 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Attempting to acquire lock 22866146310320 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Lock 22866146310320 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Attempting to release lock 22866146310320 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
+2025-07-03 14:48:36 DEBUG: Lock 22866146310320 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock
diff --git a/tmp/test_chat.py b/tmp/test_chat.py
new file mode 100644
index 0000000..d543915
--- /dev/null
+++ b/tmp/test_chat.py
@@ -0,0 +1,105 @@
+import os
+
+import torch
+
+from ctx_to_lora.model_loading import get_model, get_tokenizer
+
+# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it")
+# model = AutoModelForCausalLM.from_pretrained(
+#     "google/gemma-2-2b-it",
+#     device_map="auto",
+#     torch_dtype=torch.bfloat16,
+#     attn_implementation="sdpa",
+# )
+
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true"
+os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1"
+
+# torch.use_deterministic_algorithms(True, warn_only=True)
+torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
+torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
+torch.backends.cudnn.benchmark = False
+torch.backends.cuda.matmul.allow_tf32 = False
+torch.backends.cudnn.allow_tf32 = False
+
+# model, tokenizer = get_model_and_tokenizer(
+#     "google/gemma-3-1b-it",
+#     train=False,
+#     requires_grad=False,
+#     use_flash_attn=True,
+#     peft_config=None,
+#     model_kwargs={"attn_implementation": "flash_attention_2"},
+#     tokenizer_kwargs=None,
+#     device="cuda",
+#     dtype=torch.bfloat16,
+# )
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_kwargs = {"attn_implementation": "flash_attention_2"}
+# model_name_or_path = "Qwen/Qwen3-1.7B"
+model_name_or_path = "google/gemma-2-2b-it"
+# model_name_or_path = "google/gemma-3-1b-it"
+tokenizer = get_tokenizer(model_name_or_path, train=False)
+model = get_model(
+    model_name_or_path,
+    train=False,
+    requires_grad=False,
+    model_kwargs=model_kwargs,
+    use_flash_attn=True,
+    dtype=torch.bfloat16,
+)
+
+qa = (
+    "Architecturally, the school has a Catholic character. "
+    "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. "
+    "Immediately in front of the Main Building and facing it, "
+    'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". '
+    "Next to the Main Building is the Basilica of the Sacred Heart. "
+    "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. "
+    "It is a replica of the grotto at Lourdes, "
+    "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous "
+    "in 1858. At the end of the main drive (and in a direct line that connects through 3 "
+    "statues and the Gold Dome), is a simple, modern stone statue of Mary."
+    "\n\nAnswer the following question. "
+    "Output only the answer and do not output any other words."
+    "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?"
+)
+# x = " | ".join([str(random.randint(0, 1000)) for _ in range(800)])
+# qa = (
+#     " | ".join([str(random.randint(0, 1000)) for _ in range(10000)])
+#     + "\n\nRepeat the text above."
+# )
+
+messages = [
+    [{"role": "system", "content": ""}, {"role": "user", "content": qa}],
+    [{"role": "system", "content": ""}, {"role": "user", "content": "Hello."}],
+]
+chat_ids = tokenizer.apply_chat_template(
+    messages,
+    tokenize=True,
+    padding=True,
+    truncation=False,
+    add_special_token=False,
+    add_generation_prompt=True,
+    return_tensors="pt",
+    return_dict=True,
+).to(device)
+print(chat_ids)
+print(f"{len(chat_ids['input_ids'][0])} tokens in input")
+print(tokenizer.decode(chat_ids["input_ids"][0]))
+
+
+print(model(**chat_ids).logits)
+
+outputs = model.generate(
+    **chat_ids,
+    max_new_tokens=2**12,
+    do_sample=False,
+    temperature=0.0,
+)
+for output in outputs:
+    print(tokenizer.decode(output))
+    print("-" * 100)
+
+print(outputs)
+breakpoint()
diff --git a/tmp/test_detokenize.py b/tmp/test_detokenize.py
new file mode 100644
index 0000000..4562cb4
--- /dev/null
+++ b/tmp/test_detokenize.py
@@ -0,0 +1,35 @@
+from ctx_to_lora.model_loading import get_tokenizer
+
+model_name_or_path = "google/gemma-2-2b-it"
+tokenizer = get_tokenizer(model_name_or_path, train=True)
+
+
+qa = (
+    "Architecturally, the school has a Catholic character. "
+    "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. "
+    "Immediately in front of the Main Building and facing it, "
+    'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". '
+    "Next to the Main Building is the Basilica of the Sacred Heart. "
+    "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. "
+    "It is a replica of the grotto at Lourdes, "
+    "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous "
+    "in 1858. At the end of the main drive (and in a direct line that connects through 3 "
+    "statues and the Gold Dome), is a simple, modern stone statue of Mary."
+    "\n\nAnswer the following question. "
+    "Output only the answer and do not output any other words."
+    "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?"
+)
+
+
+messages = [{"role": "system", "content": ""}, {"role": "user", "content": qa}]
+chat_ids = tokenizer.apply_chat_template(
+    messages,
+    tokenize=True,
+    add_special_token=False,
+    add_generation_prompt=True,
+    return_tensors="pt",
+    return_dict=True,
+)
+print(chat_ids)
+print(tokenizer.batch_decode(chat_ids["input_ids"]))
+breakpoint()
diff --git a/tmp/test_emb.py b/tmp/test_emb.py
new file mode 100644
index 0000000..dd570a8
--- /dev/null
+++ b/tmp/test_emb.py
@@ -0,0 +1,51 @@
+import os
+
+import torch
+
+from ctx_to_lora.model_loading import get_model, get_tokenizer
+
+# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it")
+# model = AutoModelForCausalLM.from_pretrained(
+#     "google/gemma-2-2b-it",
+#     device_map="auto",
+#     torch_dtype=torch.bfloat16,
+#     attn_implementation="sdpa",
+# )
+
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true"
+os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1"
+
+# torch.use_deterministic_algorithms(True, warn_only=True)
+torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
+torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
+torch.backends.cudnn.benchmark = False
+torch.backends.cuda.matmul.allow_tf32 = False
+torch.backends.cudnn.allow_tf32 = False
+
+# model, tokenizer = get_model_and_tokenizer(
+#     "google/gemma-3-1b-it",
+#     train=False,
+#     requires_grad=False,
+#     use_flash_attn=True,
+#     peft_config=None,
+#     model_kwargs={"attn_implementation": "flash_attention_2"},
+#     tokenizer_kwargs=None,
+#     device="cuda",
+#     dtype=torch.bfloat16,
+# )
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_kwargs = {"attn_implementation": "flash_attention_2"}
+# model_name_or_path = "google/gemma-2-2b-it"
+model_name_or_path = "google/gemma-2-2b-it"
+tokenizer = get_tokenizer(model_name_or_path, train=False)
+model = get_model(
+    model_name_or_path,
+    train=False,
+    requires_grad=False,
+    model_kwargs=model_kwargs,
+    use_flash_attn=True,
+    dtype=torch.bfloat16,
+)
+
+breakpoint()
diff --git a/tmp/test_logits.py b/tmp/test_logits.py
new file mode 100644
index 0000000..f8799e5
--- /dev/null
+++ b/tmp/test_logits.py
@@ -0,0 +1,121 @@
+import os
+
+import torch
+
+from ctx_to_lora.model_loading import get_model, get_tokenizer
+
+# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it")
+# model = AutoModelForCausalLM.from_pretrained(
+#     "google/gemma-2-2b-it",
+#     device_map="auto",
+#     torch_dtype=torch.bfloat16,
+#     attn_implementation="sdpa",
+# )
+
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true"
+os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1"
+
+# torch.use_deterministic_algorithms(True, warn_only=True)
+torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
+torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
+torch.backends.cudnn.benchmark = False
+torch.backends.cuda.matmul.allow_tf32 = False
+torch.backends.cudnn.allow_tf32 = False
+
+# model, tokenizer = get_model_and_tokenizer(
+#     "google/gemma-3-1b-it",
+#     train=False,
+#     requires_grad=False,
+#     use_flash_attn=True,
+#     peft_config=None,
+#     model_kwargs={"attn_implementation": "flash_attention_2"},
+#     tokenizer_kwargs=None,
+#     device="cuda",
+#     dtype=torch.bfloat16,
+# )
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_kwargs = {"attn_implementation": "flash_attention_2"}
+# model_name_or_path = "google/gemma-2-2b-it"
+model_name_or_path = "google/gemma-3-1b-it"
+tokenizer = get_tokenizer(model_name_or_path, train=False)
+model = get_model(
+    model_name_or_path,
+    train=False,
+    requires_grad=False,
+    model_kwargs=model_kwargs,
+    use_flash_attn=True,
+    dtype=torch.bfloat16,
+)
+# model.config.pad_token_id = tokenizer.pad_token_id
+# if getattr(model, "generation_config", None):
+#     model.generation_config.pad_token_id = tokenizer.pad_token_id
+# print(model.name_or_path)
+
+input_text = [
+    "Write me a poem about Machine Learning.",
+    "Write me a poem about cats.",
+    "Who is Donal Trump?",
+    "Who is Mark Zuckerburg?",
+    "Tell me about Thailand.",
+    "Is Japan hot in the summer?",
+    "How computers work?",
+    "Who sells iPhone?",
+    "What exclusive games are on PS5?",
+    "Mechanical vs rubber dome keyboards",
+]
+input_ids = [tokenizer(t, return_tensors="pt").to(device) for t in input_text]
+
+padding_kwargs = dict(padding=True, padding_side="left", return_tensors="pt")
+input_ids = tokenizer.pad(
+    {k: [x[k][0] for x in input_ids] for k in input_ids[0].keys()},
+    **padding_kwargs,
+).to(device)
+print(input_ids)
+outputs = model.generate(**input_ids, do_sample=False)
+for output in outputs:
+    print(tokenizer.decode(output))
+    print("-" * 100)
+
+print(model(**input_ids).logits)
+
+# qa = (
+#     "Architecturally, the school has a Catholic character. "
+#     "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. "
+#     "Immediately in front of the Main Building and facing it, "
+#     'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". '
+#     "Next to the Main Building is the Basilica of the Sacred Heart. "
+#     "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. "
+#     "It is a replica of the grotto at Lourdes, "
+#     "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous "
+#     "in 1858. At the end of the main drive (and in a direct line that connects through 3 "
+#     "statues and the Gold Dome), is a simple, modern stone statue of Mary."
+#     "\n\nAnswer the following question. "
+#     "Output only the answer and do not output any other words."
+#     "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?"
+# )
+
+
+# messages = [{"role": "system", "content": ""}, {"role": "user", "content": qa}]
+# chat_ids = tokenizer.apply_chat_template(
+#     messages,
+#     tokenize=True,
+#     add_special_token=False,
+#     add_generation_prompt=True,
+#     return_tensors="pt",
+#     return_dict=True,
+# ).to(device)
+# print(chat_ids)
+# print(tokenizer.decode(chat_ids["input_ids"][0]))
+# outputs = model.generate(
+#     **chat_ids,
+#     max_new_tokens=100,
+#     do_sample=False,
+#     temperature=0.0,
+# )
+
+# print(model(**chat_ids).logits)
+
+# for output in outputs:
+#     print(tokenizer.decode(output))
+#     print("-" * 100)
diff --git a/tmp/test_mem_alloc.py b/tmp/test_mem_alloc.py
new file mode 100644
index 0000000..36f3087
--- /dev/null
+++ b/tmp/test_mem_alloc.py
@@ -0,0 +1,151 @@
+import os
+import time
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+def bytes_to_mib(x: int) -> float:
+    return x / (1024**2)
+
+
+def bytes_to_gib(x: int) -> float:
+    return x / (1024**3)
+
+
+def pick_dtype() -> torch.dtype:
+    if torch.cuda.is_available():
+        # Prefer bfloat16 if supported, else float16, else float32
+        if torch.cuda.is_bf16_supported():
+            return torch.bfloat16
+        return torch.float16
+    return torch.float32
+
+
+def main():
+    model_name = os.environ.get("MODEL_NAME", "google/gemma-2-2b-it")
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    dtype = pick_dtype()
+
+    if device.type != "cuda":
+        print(
+            "CUDA is not available; this script is intended for GPU memory measurements."
+        )
+        return
+
+    torch.cuda.empty_cache()
+    torch.cuda.synchronize()
+
+    # Load tokenizer on CPU (doesn't affect CUDA mem)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    # Measure peak memory during model load
+    torch.cuda.reset_peak_memory_stats(device)
+    load_start = time.time()
+
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=dtype,
+        device_map=None,  # load on CPU first, then move to desired single device
+    )
+    model.to(device)
+    model.eval()
+
+    torch.cuda.synchronize()
+    model_load_peak = torch.cuda.max_memory_allocated(device)
+    load_elapsed = time.time() - load_start
+
+    # Baseline after load
+    baseline_alloc = torch.cuda.memory_allocated(device)
+
+    # Prepare a chat-style prompt if available; fallback to plain text
+    doc = """
+Matryoshka dolls (Russian: матрёшка, romanized: matryoshka/ˌmætriˈɒʃkə/), also known as stacking dolls, nesting dolls, Russian tea dolls, or Russian dolls,[1] are a set of wooden dolls of decreasing size placed one inside another. The name Matryoshka is a diminutive form of Matryosha (Матрёша), in turn a hypocorism of the Russian female first name Matryona (Матрёна).[2]
+
+A set of matryoshkas consists of a wooden figure, which separates at the middle, top from bottom, to reveal a smaller figure of the same sort inside, which has, in turn, another figure inside of it, and so on.
+
+The first Russian nested doll set was made in 1890 by woodturning craftsman and wood carver Vasily Zvyozdochkin from a design by Sergey Malyutin, who was a folk crafts painter at Abramtsevo. Traditionally the outer layer is a woman, dressed in a Russian sarafan dress. The figures inside may be of any gender; the smallest, innermost doll is typically a baby turned from a single piece of wood. Much of the artistry is in the painting of each doll, which can be very elaborate. The dolls often follow a theme; the themes may vary, from fairy tale characters to Soviet leaders. In some countries, matryoshka dolls are often referred to as babushka dolls, though they are not known by this name in Russian; babushka (бабушка) means 'grandmother; old woman'.[3]
+History
+The original matryoshka set by Zvyozdochkin and Malyutin, 1892
+
+The first Russian nested doll set was carved in 1890 at the Children's Education Workshop by Vasily Zvyozdochkin and designed by Sergey Malyutin, who was a folk crafts painter in the Abramtsevo estate of Savva Mamontov, a Russian industrialist and patron of arts.[4][5] Mamontov's brother, Anatoly Ivanovich Mamontov (1839–1905), created the Children's Education Workshop to make and sell children's toys. The doll set was painted by Malyutin. Malyutin's doll set consisted of eight dolls—the outermost was a mother in a traditional dress holding a red-combed rooster. The inner dolls were her children, girls and a boy, and the innermost a baby. The Children's Education Workshop was closed in the late 1890s, but the tradition of the matryoshka simply relocated to Sergiyev Posad, the Russian city known as a toy-making center since the fourteenth century.[6][4]
+
+The inspiration for matryoshka dolls is not clear. Matryoshka dolls may have been inspired by a nesting doll imported from Japan.[5][7] The Children's Education workshop where Zvyozdochkin was a lathe operator received a five piece, cylinder-shaped nesting doll featuring Fukuruma (Fukurokuju) in the late 1890s,[8] which is now part of the collection at the Sergiev Posad Museum of Toys.[8] Other east Asian dolls share similarities with matryoshka dolls such as the Kokeshi dolls,[4][9] originating in Northern Honshū, the main island of Japan, although they cannot be placed one inside another, and the round hollow daruma doll depicting a Buddhist monk.[9][10] Another possible source of inspiration is the nesting Easter eggs produced on a lathe by Russian woodworkers during the late 19th Century.[3][11]
+
+Savva Mamontov's wife presented a set of matryoshka dolls at the Exposition Universelle in Paris in 1900, and the toy earned a bronze medal. Soon after, matryoshka dolls were being made in several places in Russia and shipped around the world. 
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": doc + "\n\nSummarize the article.",
+        },
+    ]
+
+    prompt_text = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+
+    inputs = tokenizer(prompt_text, return_tensors="pt").to(device)
+
+    # Optional quick warmup to stabilize kernels (doesn't affect peak calc since we reset after)
+    with torch.no_grad():
+        _ = model.generate(**inputs, max_new_tokens=1)
+        torch.cuda.synchronize()
+
+    # Measure generation peak memory for 512 new tokens
+    torch.cuda.reset_peak_memory_stats(device)
+    gen_start = time.time()
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=int(os.environ.get("MAX_NEW_TOKENS", 512)),
+            do_sample=False,
+            temperature=None,
+            top_p=None,
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+    torch.cuda.synchronize()
+    gen_elapsed = time.time() - gen_start
+
+    gen_peak_total = torch.cuda.max_memory_allocated(
+        device
+    )  # includes baseline present at reset
+    gen_additional = max(0, gen_peak_total - baseline_alloc)
+
+    decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+    # Print summary
+    print("=== Memory allocation summary (CUDA) ===")
+    print(f"Device: {device} | {torch.cuda.get_device_name(device)}")
+    print(f"Model: {model_name}")
+    print(f"DType: {dtype}")
+    print()
+    print("Model load:")
+    print(
+        f"  Peak allocated during load: {bytes_to_mib(model_load_peak):.2f} MiB ({bytes_to_gib(model_load_peak):.2f} GiB)"
+    )
+    print(f"  Time to load + move to device: {load_elapsed:.2f}s")
+    print(
+        f"  Baseline after load (current allocated): {bytes_to_mib(baseline_alloc):.2f} MiB ({bytes_to_gib(baseline_alloc):.2f} GiB)"
+    )
+    print()
+    print("Generation (max_new_tokens=512):")
+    print(
+        f"  Peak allocated total during generation: {bytes_to_mib(gen_peak_total):.2f} MiB ({bytes_to_gib(gen_peak_total):.2f} GiB)"
+    )
+    print(
+        f"  Additional peak over baseline during generation: {bytes_to_mib(gen_additional):.2f} MiB ({bytes_to_gib(gen_additional):.2f} GiB)"
+    )
+    print(f"  Time to generate: {gen_elapsed:.2f}s")
+    print()
+    # Response preview
+    preview = decoded
+    if len(preview) > 800:
+        preview = preview[:800] + "\n... [truncated]"
+    print("=== Response preview ===")
+    print(preview)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tmp/test_pad.py b/tmp/test_pad.py
new file mode 100644
index 0000000..4ebea5f
--- /dev/null
+++ b/tmp/test_pad.py
@@ -0,0 +1,8 @@
+import torch
+from transformers import AutoTokenizer
+
+tk = AutoTokenizer.from_pretrained("google/gemma-2-2b-it")
+x = [torch.rand(5, 3), torch.rand(10, 3), torch.rand(7, 3)]
+padded_x = tk.pad({"input_ids": x}, padding=True)
+print(padded_x)
+breakpoint()
diff --git a/tmp/torch_profiler_test.py b/tmp/torch_profiler_test.py
new file mode 100644
index 0000000..73d72d6
--- /dev/null
+++ b/tmp/torch_profiler_test.py
@@ -0,0 +1,69 @@
+# (c) Meta Platforms, Inc. and affiliates.
+import logging
+import socket
+from datetime import datetime
+
+import torch
+from torch.autograd.profiler import record_function
+from torchvision import models
+
+logging.basicConfig(
+    format="%(levelname)s:%(asctime)s %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger: logging.Logger = logging.getLogger(__name__)
+logger.setLevel(level=logging.INFO)
+
+TIME_FORMAT_STR: str = "%b_%d_%H_%M_%S"
+
+
+def trace_handler(prof: torch.profiler.profile):
+    # Prefix for file names.
+    host_name = socket.gethostname()
+    timestamp = datetime.now().strftime(TIME_FORMAT_STR)
+    file_prefix = f"{host_name}_{timestamp}"
+
+    # Construct the trace file.
+    prof.export_chrome_trace(f"{file_prefix}.json.gz")
+
+    # Construct the memory timeline file.
+    prof.export_memory_timeline(f"{file_prefix}.html", device="cuda:0")
+
+
+def run_resnet50(num_iters=5, device="cuda:0"):
+    model = models.resnet50().to(device=device)
+    inputs = torch.randn(1, 3, 224, 224, device=device)
+    labels = torch.rand_like(model(inputs))
+    optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)
+    loss_fn = torch.nn.CrossEntropyLoss()
+
+    with torch.profiler.profile(
+        activities=[
+            torch.profiler.ProfilerActivity.CPU,
+            torch.profiler.ProfilerActivity.CUDA,
+        ],
+        schedule=torch.profiler.schedule(wait=0, warmup=0, active=6, repeat=1),
+        record_shapes=True,
+        profile_memory=True,
+        with_stack=True,
+        on_trace_ready=trace_handler,
+    ) as prof:
+        for _ in range(num_iters):
+            prof.step()
+            with record_function("## forward ##"):
+                pred = model(inputs)
+
+            with record_function("## backward ##"):
+                loss_fn(pred, labels).backward()
+
+            with record_function("## optimizer ##"):
+                optimizer.step()
+                optimizer.zero_grad(set_to_none=True)
+
+
+if __name__ == "__main__":
+    # Warm up
+    run_resnet50()
+    # Run the resnet50 model
+    run_resnet50()
diff --git a/tmp/visualize_ds_len.py b/tmp/visualize_ds_len.py
new file mode 100644
index 0000000..8d40598
--- /dev/null
+++ b/tmp/visualize_ds_len.py
@@ -0,0 +1,101 @@
+import os
+from functools import partial
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from ctx_to_lora.data.processing import get_tokenized_dataset
+from ctx_to_lora.model_loading import get_tokenizer
+from ctx_to_lora.utils import check_is_iterable
+
+tokenizer_kwargs = {"max_length": 2**13}
+ctx_tokenizer_kwargs = {"max_length": 2**13}  # not used for now
+tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True)
+ctx_tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True)
+
+_get_tokenized_dataset = partial(
+    get_tokenized_dataset,
+    base_model_max_len=2**13,
+    tokenizer=tokenizer,
+    tokenizer_kwargs=tokenizer_kwargs,
+    ctx_model_max_len=2**13,
+    ctx_tokenizer=ctx_tokenizer,
+    ctx_tokenizer_kwargs=ctx_tokenizer_kwargs,
+    add_ctx_to_chat=False,
+    add_repeat_prompt=False,
+    add_negative_prompt=False,
+    use_kl_loss=False,
+    # streaming=data_args.streaming,
+)
+
+train_ds_names = [
+    # "fw_qa",
+    # "fw_qa_large",
+    # "ctx_qa",
+    # "pwc",
+    # "hotpot_qa",
+    # "squad",
+    # "drop",
+    # "narrativeqa",
+    # "quoref",
+    # "ropes",
+    # "synthetic_convqa",
+    # "booksum",
+    # "gov_report",
+    # "wikitext-2",
+    # "wikitext-103",
+    # "fw_qa_2"
+    # "fw_qa_xl",
+    # "fw_qa_3_medium",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_0_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_1_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_3_tiny",
+]
+tokenized_ds = dict()
+val_ds_names = []  # ["fw_qa_large", "ctx_qa", "pwc", "hotpot_qa", "squad"]
+for split, ds_names in zip(["train", "validation"], [train_ds_names, val_ds_names]):
+    tokenized_ds[split] = {
+        os.path.basename(ds_name): _get_tokenized_dataset(ds_name, split)
+        for ds_name in ds_names
+    }
+    for ds_name in tokenized_ds[split]:
+        print(ds_name)
+        print(f"Num samples: {len(tokenized_ds[split][ds_name])}")
+        sample = tokenized_ds[split][ds_name][0]
+
+        # TODO: let's investigate samples that are very long and very short
+        if check_is_iterable(sample["input_ids"]):
+            tokens = []
+            for sample in tokenized_ds[split][ds_name]:
+                tokens += [len(ids) for ids in sample["input_ids"]]
+        else:
+            tokens = [
+                len(sample["input_ids"]) for sample in tokenized_ds[split][ds_name]
+            ]
+
+        print(f"Num avg. tokens: {sum(tokens) / len(tokens)}")
+        print(f"Num max. tokens: {max(tokens)}")
+        print(f"Num min. tokens: {min(tokens)}")
+        print(f"Num std. tokens: {np.std(tokens)}")
+        plt.hist(tokens, bins=100)
+        plt.savefig(f"tmp/n_token_hist_{split}_{ds_name}.png")
+        plt.close()
+
+        long_indices = [i for i, n_tokens in enumerate(tokens) if n_tokens > 100]
+        short_indices = [i for i, n_tokens in enumerate(tokens) if n_tokens < 30]
+        breakpoint()
+        for i in long_indices:
+            print(f"Long sample {i}: {tokenized_ds[split][ds_name][i]}")
+        breakpoint()
+        for i in short_indices:
+            print(f"Short sample {i}: {tokenized_ds[split][ds_name][i]}")
+
+        ctx_tokens = [len(sample["ctx_ids"]) for sample in tokenized_ds[split][ds_name]]
+        print(f"Num avg. ctx tokens: {sum(ctx_tokens) / len(ctx_tokens)}")
+        print(f"Num max. ctx tokens: {max(ctx_tokens)}")
+        print(f"Num min. ctx tokens: {min(ctx_tokens)}")
+        print(f"Num std. ctx tokens: {np.std(ctx_tokens)}")
+        plt.hist(ctx_tokens, bins=100)
+        plt.savefig(f"tmp/ctx_n_token_hist_{split}_{ds_name}.png")
+        plt.close()
diff --git a/tmp/visualize_long_samples.py b/tmp/visualize_long_samples.py
new file mode 100644
index 0000000..0104a43
--- /dev/null
+++ b/tmp/visualize_long_samples.py
@@ -0,0 +1,133 @@
+import os
+from functools import partial
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from ctx_to_lora.data.processing import get_tokenized_dataset, load_and_process_dataset
+from ctx_to_lora.model_loading import get_tokenizer
+
+tokenizer_kwargs = {"max_length": 2**13}
+ctx_tokenizer_kwargs = {"max_length": 2**13}  # not used for now
+tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True)
+ctx_tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True)
+
+_get_tokenized_dataset = partial(
+    get_tokenized_dataset,
+    base_model_max_len=2**13,
+    tokenizer=tokenizer,
+    tokenizer_kwargs=tokenizer_kwargs,
+    ctx_model_max_len=2**13,
+    ctx_tokenizer=ctx_tokenizer,
+    ctx_tokenizer_kwargs=ctx_tokenizer_kwargs,
+    add_ctx_to_chat=False,
+    add_repeat_prompt=False,
+    add_negative_prompt=False,
+    use_kl_loss=False,
+    # streaming=data_args.streaming,
+)
+_load_and_process_dataset = partial(
+    load_and_process_dataset,
+    add_negative_prompt=False,
+    add_repeat_prompt=False,
+    repeat_prob=0,
+    is_pretrain=False,
+    streaming=False,
+    num_proc=8,
+)
+
+train_ds_names = [
+    # "fw_qa",
+    # "fw_qa_large",
+    # "ctx_qa",
+    # "pwc",
+    # "hotpot_qa",
+    # "squad",
+    # "drop",
+    # "narrativeqa",
+    # "quoref",
+    # "ropes",
+    # "synthetic_convqa",
+    # "booksum",
+    # "gov_report",
+    # "wikitext-2",
+    # "wikitext-103",
+    # "fw_qa_2"
+    # "fw_qa_xl",
+    # "fw_qa_3_medium",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_0_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_1_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny",
+    "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_3_tiny",
+]
+loaded_dataset = dict()
+val_ds_names = []  # ["fw_qa_large", "ctx_qa", "pwc", "hotpot_qa", "squad"]
+for split, ds_names in zip(["train", "validation"], [train_ds_names, val_ds_names]):
+    loaded_dataset[split] = {
+        os.path.basename(ds_name): _load_and_process_dataset(ds_name, split)
+        for ds_name in ds_names
+    }
+    for ds_name in loaded_dataset[split]:
+        print(ds_name)
+        print(f"Num samples: {len(loaded_dataset[split][ds_name])}")
+        ds = loaded_dataset[split][ds_name]
+        sample = loaded_dataset[split][ds_name][0]
+
+        # TODO: let's investigate samples that are very long and very short
+        if "responses" in sample:
+            char_lens = []
+            responses = []
+            questions = []
+            ctxs = []
+            for sample in loaded_dataset[split][ds_name]:
+                char_lens += [len(res) for res in sample["responses"]]
+                responses += sample["responses"]
+                questions += sample["prompts"]
+                ctxs += [sample["context"]] * len(sample["responses"])
+        else:
+            responses = ds["response"]
+            questions = ds["prompt"]
+            ctxs = ds["context"]
+
+            char_lens = [
+                len(sample["response"]) for sample in loaded_dataset[split][ds_name]
+            ]
+
+        print(f"Num avg. char_lens: {sum(char_lens) / len(char_lens)}")
+        print(f"Num max. char_lens: {max(char_lens)}")
+        print(f"Num min. char_lens: {min(char_lens)}")
+        print(f"Num std. char_lens: {np.std(char_lens)}")
+        plt.hist(char_lens, bins=100)
+        plt.savefig(f"tmp/n_token_hist_{split}_{ds_name}.png")
+        plt.close()
+
+        long_indices = [i for i, clen in enumerate(char_lens) if clen > 2000]
+        short_indices = [i for i, clen in enumerate(char_lens) if clen < 100]
+        breakpoint()
+        for i in long_indices:
+            print(f"Long sample {i}")
+            print(f"Response length: {len(responses[i])}")
+            print(f"Context: {ctxs[i]}")
+            print(f"Question: {questions[i]}")
+            print(f"Response: {responses[i]}")
+            print("=" * 80)
+
+        breakpoint()
+        for i in short_indices:
+            print(f"Short sample {i}")
+            print(f"Response length: {len(responses[i])}")
+            print(f"Context: {ctxs[i]}")
+            print(f"Question: {questions[i]}")
+            print(f"Response: {responses[i]}")
+            print("=" * 80)
+
+        ctx_tokens = [
+            len(sample["ctx_ids"]) for sample in loaded_dataset[split][ds_name]
+        ]
+        print(f"Num avg. ctx char_lens: {sum(ctx_tokens) / len(ctx_tokens)}")
+        print(f"Num max. ctx char_lens: {max(ctx_tokens)}")
+        print(f"Num min. ctx char_lens: {min(ctx_tokens)}")
+        print(f"Num std. ctx char_lens: {np.std(ctx_tokens)}")
+        plt.hist(ctx_tokens, bins=100)
+        plt.savefig(f"tmp/ctx_n_token_hist_{split}_{ds_name}.png")
+        plt.close()