import dataclasses import os import sys from dataclasses import dataclass, field from enum import Enum, auto from typing import Any, Dict, List, Literal, NewType, Optional, Tuple import yaml from transformers import MODEL_FOR_CAUSAL_LM_MAPPING, HfArgumentParser from modeling_utils import AGGREGATOR_TYPE MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) DataClassType = NewType("DataClassType", Any) class ArgumentParser(HfArgumentParser): def parse_yaml_and_args( self, yaml_arg: str, other_args: Optional[list[str]] = None ) -> list[dataclass]: """ Parse a YAML file and overwrite the default/loaded values with the values provided to the command line. Args: yaml_arg (`str`): The path to the config file used other_args (`List[str]`, *optional`): A list of strings to parse as command line arguments, e.g. ['--arg=val', '--arg2=val2']. Returns: [`List[dataclass]`]: a list of dataclasses with the values from the YAML file and the command line """ arg_list = self.parse_yaml_file(os.path.abspath(yaml_arg)) outputs = [] # strip other args list into dict of key-value pairs other_args = { arg.split("=")[0].strip("-"): arg.split("=")[1] for arg in other_args } used_args = {} # overwrite the default/loaded value with the value provided to the command line # adapted from https://github.com/huggingface/transformers/blob/d0b5002378daabf62769159add3e7d66d3f83c3b/src/transformers/hf_argparser.py#L327 for data_yaml, data_class in zip(arg_list, self.dataclass_types): keys = {f.name for f in dataclasses.fields(data_yaml) if f.init} inputs = {k: v for k, v in vars(data_yaml).items() if k in keys} for arg, val in other_args.items(): # add only if in keys if arg in keys: base_type = data_yaml.__dataclass_fields__[arg].type inputs[arg] = val # cast type for ints, floats (default to strings) if base_type in [int, float]: inputs[arg] = base_type(val) if base_type == list[str]: inputs[arg] = [str(v) for v in val.split(",")] # bool of a non-empty string is True, so we manually check for bools if base_type == bool: if val in ["true", "True"]: inputs[arg] = True else: inputs[arg] = False if base_type == dict: inputs[arg] = yaml.load(val, Loader=yaml.FullLoader) # add to used-args so we can check if double add if arg not in used_args: used_args[arg] = val else: raise ValueError( f"Duplicate argument provided: {arg}, may cause unexpected behavior" ) # else: # raise ValueError(f"Argument provided not found in dataclass: {arg}") obj = data_class(**inputs) outputs.append(obj) return outputs def parse(self) -> DataClassType | tuple[DataClassType]: if len(sys.argv) == 2 and sys.argv[1].endswith(".yaml"): # If we pass only one argument to the script and it's the path to a YAML file, # let's parse it to get our arguments. output = self.parse_yaml_file(os.path.abspath(sys.argv[1].split("=")[-1])) # parse command line args and yaml file elif len(sys.argv) > 2 and sys.argv[1].endswith(".yaml"): output = self.parse_yaml_and_args( os.path.abspath(sys.argv[1].split("=")[-1]), sys.argv[2:] ) # parse --config for the yaml path and other command line args elif any([arg.startswith("--config") for arg in sys.argv]): yaml_arg = [ arg for arg in sys.argv[1:] if arg.startswith("--config") and arg.endswith(".yaml") ][0] other_args = [arg for arg in sys.argv[1:] if arg != yaml_arg] output = self.parse_yaml_and_args( os.path.abspath(yaml_arg.split("=")[-1]), other_args ) # parse command line args only else: output = self.parse_args_into_dataclasses() if len(output) == 1: output = output[0] return output class ExperimentSetup(str, Enum): LORA = "lora" HYPER_LORA = "hyper_lora" FULL_FINETUNE = "full_finetune" @dataclass class ModelArguments: """ Arguments for the base model. """ model_name_or_path: str = field( default=None, metadata={"help": ("Base model name or path.")}, ) # use_peft: bool = field( # default=False, # metadata={"help": ("Whether to use PEFT or not for training.")}, # ) @dataclass class LoRAArguments: lora_r: Optional[int] = field( default=8, metadata={"help": ("LoRA R value.")}, ) lora_dropout: Optional[float] = field( default=0.05, metadata={"help": ("LoRA dropout.")}, ) target_modules: Optional[list[str]] = field( default=None, metadata={"help": ("LoRA target modules.")}, ) @dataclass class CtxTrainingArguments: exp_setup: ExperimentSetup = field( default=ExperimentSetup.LORA, metadata={"help": "Experiment setup - LoRA, HyperLoRA, or full finetuning"}, ) max_base_len: Optional[int] = field( default=2**13, metadata={"help": "Maximum base length for training."}, ) aggregator_type: AGGREGATOR_TYPE = field( default=AGGREGATOR_TYPE.POOLER, metadata={"help": "Aggregator type for HyperLoRA."}, ) @dataclass class DataArguments: train_ds_name: str = field( default="", metadata={"help": "Name of the training dataset."}, ) if __name__ == "__main__": print(ExperimentSetup) print(ExperimentSetup.LORA) print(ExperimentSetup.HYPER_LORA) print(ExperimentSetup.FULL_FINETUNE)