apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

gen script 16k context length

Browse source
This commit is contained in:
51616 2025-04-28 02:38:29 +00:00
parent 26841bda04
commit d168919fd9
1 changed files with 9 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

14
generate_fw_edu_qa_vllm.py
View file

@ -109,7 +109,7 @@ if __name__ == "__main__":
# base_url=f"http://{vllm_ip}:{vllm_port}/v1",
# api_key=api_key,
# )
mm_kwargs = {"image":0} # if "gemma-3" in vllm_model else {}
mm_kwargs = {"image": 0} # if "gemma-3" in vllm_model else {}
llm_kwargs = dict(
model=vllm_model,
dtype="bfloat16",
@ -117,12 +117,12 @@ if __name__ == "__main__":
# config_format="mistral",
# load_format="mistral",
enable_prefix_caching=True,
max_model_len=11264,#2**14,
max_model_len=2**14, # 11264,
limit_mm_per_prompt=mm_kwargs,
# enable_chunked_prefill=True,
)
if vllm_model=="mistralai/Mistral-Small-3.1-24B-Instruct-2503":
if vllm_model == "mistralai/Mistral-Small-3.1-24B-Instruct-2503":
llm_kwargs["tokenizer_mode"] = "mistral"
llm_kwargs["config_format"] = "mistral"
llm_kwargs["load_format"] = "mistral"
@ -161,7 +161,11 @@ if __name__ == "__main__":
messages,
sampling_params=SamplingParams(
max_tokens=2048,
temperature=0.1 if vllm_model=="mistralai/Mistral-Small-3.1-24B-Instruct-2503" else 0.7,
temperature=(
0.1
if vllm_model == "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
else 0.7
),
# frequency_penalty=0.1,
),
)