2025-09-30 14:53:21 +00:00
|
|
|
# main results
|
|
|
|
|
# batched
|
2026-06-15 04:31:47 +00:00
|
|
|
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path train_outputs/runs/$RUN_NAME/checkpoint-$step/pytorch_model.bin --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e longbench/gov_report_e --split test --max_ctx_chunk_len 8192 --eval_batch_size_gen 1
|
2025-09-30 14:53:21 +00:00
|
|
|
|
|
|
|
|
# iterative
|
2026-06-15 04:31:47 +00:00
|
|
|
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path train_outputs/runs/$RUN_NAME/checkpoint-$step/pytorch_model.bin --datasets squad drop ropes longbench/qasper_e longbench/2wikimqa_e longbench/multifieldqa_en_e longbench/gov_report_e --split test --max_ctx_chunk_len 8192 --eval_batch_size_gen 1 --use_iterative_mode
|
2025-09-30 14:53:21 +00:00
|
|
|
|
|
|
|
|
# query internalization
|
2025-10-01 13:11:54 +00:00
|
|
|
WANDB_MODE=disabled uv run run_eval.py --checkpoint_path train_outputs/runs/$RUN_NAME/checkpoint-$step/pytorch_model.bin --datasets squad --split test --eval_batch_size_gen=1 --flip_ctx_inp
|
2025-09-30 14:53:21 +00:00
|
|
|
|
|
|
|
|
# replaced squad context
|
2025-10-01 13:11:54 +00:00
|
|
|
WANDB_MODE=disabled uv run python run_eval.py --checkpoint_path train_outputs/runs/$RUN_NAME/checkpoint-$step/pytorch_model.bin --datasets squad_assistant_ctx_no_passage --split test
|
|
|
|
|
WANDB_MODE=disabled uv run python run_eval.py --checkpoint_path train_outputs/runs/$RUN_NAME/checkpoint-$step/pytorch_model.bin --datasets squad_negative_no_passage --split test
|
