o Zhr@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZddlmZddlmZddlmZmZmZmZmZddlmZddlm Z!ddl"Z#ddl$Z$ddl%m&Z'dd l(m)Z)m*Z*m+Z+dd l,m-Z-dd l$m.Z.dd l/m0Z0m1Z1m2Z2m3Z3m4Z4dd l5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mZ>m?Z?ddl@mAZAddlBmCZCddlDmEZEmFZFddlGmHZHddlImJZJmKZKmLZLddlMmNZNddlOmPZPddlQmRZRmSZSmTZTddlUmVZVmWZWddlXmYZYmZZZddl[m\Z\ddl]m^Z^m_Z_ddl`maZaddlbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjddlkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZdd lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd!lmZmZmZdd"l mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd#lmZdd$lmZdd%lmZedgZegZerdd&lmZeZerdd'lmZer#ddlZer\ddlmmZddlmmZddlmZdd lm6Ze-eޡe-ekZer[ddlm&mZnd(Zerddlmm$Zdd lm6Ze-ee-d)kZdd*lkmZmZmZmZnd(ZerddlZerdd+lmZerdd,lmZmZdd lm6Zdd-lmZdd.lmZmZmZmZmZmZmZe3gZe-ee-d/krdd0lmZe-ee-d1krdd2lmZeeg7ZeLrdd3lmZed4rdd5lmZd6d7Zd8d9Zd:d;Zer2ddl Z er2ddlZeÐ e Z d<Z d=Zd>Zd?Zd@ZdAZdBZedCdDGdEdFdFZdS)Guc The Trainer class, to easily train a 🤗 Transformers from scratch or finetune it on a new task. N)Mappingpartial)Path) TYPE_CHECKINGAnyCallableOptionalUnion)#get_reporting_integration_callbacks) ModelCard create_repo upload_folder)version)nn) DataLoaderDatasetIterableDataset RandomSamplerSequentialSampler) __version__)PretrainedConfig) DataCollatorDataCollatorWithPaddingdefault_data_collator) DebugOptionDebugUnderflowOverflow)SequenceFeatureExtractor)FeatureExtractionMixin)"ALL_HYPERPARAMETER_SEARCH_BACKENDSdefault_hp_search_backend)BaseImageProcessor)deepspeed_initdeepspeed_load_checkpointis_deepspeed_available)tpu_spmd_dataloader)TrainingSummary)PreTrainedModelload_sharded_checkpoint unwrap_model)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESMODEL_MAPPING_NAMES) Adafactor get_scheduler)ProcessorMixin)ALL_LAYERNORM_LAYERS"is_torch_greater_or_equal_than_2_3)PreTrainedTokenizerBase)CallbackHandlerDefaultFlowCallbackExportableStatePrinterCallbackProgressCallbackTrainerCallbackTrainerControl TrainerState)DistributedTensorGathererEvalLoopContainerIterableDatasetShard LabelSmootherLayerWiseDummyOptimizerLengthGroupedSamplerSequentialDistributedSamplerdistributed_broadcast_scalarsdistributed_concatfind_batch_sizeget_model_param_countget_module_class_from_nameget_parameter_names nested_concat nested_detachnested_numpifynested_xla_mesh_reducereissue_pt_warningsremove_dummy_checkpointset_rng_state_for_device)PREFIX_CHECKPOINT_DIRBestRunEvalLoopOutputEvalPredictionHPSearchBackend HubStrategyPredictionOutputRemoveColumnsCollator SaveStrategyTrainerMemoryTracker TrainOutputcheck_target_module_existsdefault_compute_objectivedenumpify_detensorizeenable_full_determinismfind_executable_batch_sizeget_last_checkpoint has_lengthneftune_post_forward_hooknumber_of_arguments seed_workerset_seed speed_metrics)OptimizerNames ParallelModeTrainingArguments))ADAPTER_CONFIG_NAMEADAPTER_SAFE_WEIGHTS_NAMEADAPTER_WEIGHTS_NAME CONFIG_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEXLA_FSDPV2_MIN_VERSIONPushInProgressPushToHubMixincan_return_losscheck_torch_load_is_safe find_labelsis_accelerate_availableis_apex_availableis_apollo_torch_availableis_bitsandbytes_availableis_datasets_availableis_galore_torch_availableis_grokadamw_availableis_in_notebookis_ipex_availableis_liger_kernel_availableis_lomo_availableis_peft_availableis_safetensors_availableis_sagemaker_dp_enabledis_sagemaker_mp_enabledis_schedulefree_availableis_torch_hpu_availableis_torch_mlu_availableis_torch_mps_availableis_torch_musa_availableis_torch_neuroncore_availableis_torch_npu_availableis_torch_xla_availableis_torch_xpu_availableis_torchao_availablelogging strtobool)deprecate_kwarg)requires)QuantizationMethod)NotebookProgressCallback)ampFz1.10)smp_forward_backwardsmp_forward_only smp_gathersmp_nested_concat) PeftModel) Acceleratorskip_first_batches)AcceleratorState)AutocastKwargsDistributedDataParallelKwargsDistributedTypeload_fsdp_modelload_fsdp_optimizersave_fsdp_modelsave_fsdp_optimizer1.3.0)TorchTensorParallelPluginz0.23.0)SeedableRandomSampler)DeepSpeedSchedulerWrapper0.28.0)DataLoaderConfigurationcCsPtr&tf}ttjdtdkr!ddlm}g||R}t||SdS)Npeftz0.7.0r)PeftMixedModelF) rrrparse importlibmetadatarr isinstance)modelZclasses_to_checkrrC/var/www/auris/lib/python3.10/site-packages/transformers/trainer.py_is_peft_models  rcCs&trdtttjvrddiSiS)NZ adapter_onlyT)rwlistinspect signaturer parametersrrrr_get_fsdp_ckpt_kwargssrcCsxttjjtdjkrtSttjtdkrtjntj }|j j tj tj g}|tt tjg7}tj|S)Nz2.6z2.0.0)rrtorchrrelease contextlib nullcontextnpZ_corecoreZ multiarray _reconstructndarraydtypetypeZuint32Z serialization safe_globals)Znp_coreZ allowlistrrrrs " rztraining_args.binztrainer_state.jsonz optimizer.ptz scaler.ptz optimizer.binz scheduler.ptZpytorch_model_fsdp)r accelerate)backendsc@s eZdZdZddlmZmZmZmZm Z e ddddd dd e e e jd fd ed eedee eedfdee eeeefdfdee eeeefdeege fdeedeeegefdeeedeeej j!eej j"j#fdeee$ej j!eee%ffdeeej&ej&gej&ffddZ'e(deefddZ)e)j*dddZ)ddZ+d d!Z,d"d#Z-d$d%Z.d&d'Z/d(d)Z0d*d+Z1dd,dd-eefd.d/Z2 dd ed-eedefd0d1Z3ddeedeej4j5j6fd2d3Z7 4 dd,ed-ed5e8d6eeegej4j5j6fd7e9d8eede:fd9d:Z;de:fd;d<ZZ=ddee eefde:fd?d@Z>dAede:fdBdCZ?dDe8fdEdFZ@deefdGdHZAdIdJZBdKdLZCdMdNZDddOee eej jEjFffdPdQZGeH dd ed ee dee%e%ffdRdSZIddDe8dTej j!fdUdVZJdWe:de8fdXdYZKeHddZe:d[ee8de8fd\d]ZLd^e d_eee%fffd`daZMd^e d_eee%ffdbe8dceeeNffdddeZOdfefdgdhZPddidjZQddkdlZRd4ejSfdmdnZTdodpZUddqdrZV ddsee ee9fd^e d_eee%fd fdteeefdudvZW ddwdxZXdydzZYdd{d|ZZd}d~Z[ddZ\dddZ] dddZ^ddZ_ddZ`ddZaddZbddZcddZdddZeddZfddZg ddeed_geeeNffdeeeeeNfgeNfde8de eeefdee dehfdeed_gefde eieeiffddZjddeeeNfdeeNdd fddZkde ej&e%fde ej&e%ffddZldeee ej&e%ffdeee ej&e%fffddZmddZnd dee9fddZo dd e jdeee ej&e%ffdej&fddZpd ddZqde9fddZrde9fddZsd deede9fddZtddeefddZud deefddZvdd„Zwd exd4fdeefddĄZyd dddƄZz ǐd dee eeeeffdeeededeeeNffdd˄Z{ ̐d dAedeeedede|fdd΄Z} ǐd dWe:d-edee9deeedede~f ddфZdddӄZ dd e jdeee ej&e%ffde9deeedeeej&eej&eej&ff ddՄZdeee ej&e%fffddׄZddeefddڄZ ddeedeede eeed fdeedeede eeed fde eeed fd,e eeed fde eeed ffddZddZddZ  ddeede9deedeedef ddZ ǐd dWe:d-edee9deeedede~f ddZddZdddZddZdddZddZddZd edWe:de8fddZd S(TraineruA Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers. Args: model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*): The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed. [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers models. args ([`TrainingArguments`], *optional*): The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided. data_collator (`DataCollator`, *optional*): The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will default to [`default_data_collator`] if no `processing_class` is provided, an instance of [`DataCollatorWithPadding`] otherwise if the processing_class is a feature extractor or tokenizer. train_dataset (Union[`torch.utils.data.Dataset`, `torch.utils.data.IterableDataset`, `datasets.Dataset`], *optional*): The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally sets the seed of the RNGs used. eval_dataset (Union[`torch.utils.data.Dataset`, Dict[str, `torch.utils.data.Dataset`, `datasets.Dataset`]), *optional*): The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each dataset prepending the dictionary key to the metric name. processing_class (`PreTrainedTokenizerBase` or `BaseImageProcessor` or `FeatureExtractionMixin` or `ProcessorMixin`, *optional*): Processing class used to process the data. If provided, will be used to automatically process the inputs for the model, and it will be saved along the model to make it easier to rerun an interrupted training or reuse the fine-tuned model. This supersedes the `tokenizer` argument, which is now deprecated. model_init (`Callable[[], PreTrainedModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start from a new instance of the model as given by this function. The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to be able to choose different architectures according to hyper parameters (such as layer count, sizes of inner layers, dropout probabilities etc). compute_loss_func (`Callable`, *optional*): A function that accepts the raw model outputs, labels, and the number of items in the entire accumulated batch (batch_size * gradient_accumulation_steps) and returns the loss. For example, see the default [loss function](https://github.com/huggingface/transformers/blob/052e652d6d53c2b26ffde87e039b723949a53493/src/transformers/trainer.py#L3618) used by [`Trainer`]. compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*): The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return a dictionary string to metric values. *Note* When passing TrainingArgs with `batch_eval_metrics` set to `True`, your compute_metrics function must take a boolean `compute_result` argument. This will be triggered after the last eval batch to signal that the function needs to calculate and return the global summary statistics rather than accumulating the batch-level statistics callbacks (List of [`TrainerCallback`], *optional*): A list of callbacks to customize the training loop. Will add those to the list of default callbacks detailed in [here](callback). If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method. optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`): A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`. optimizer_cls_and_kwargs (`Tuple[Type[torch.optim.Optimizer], Dict[str, Any]]`, *optional*): A tuple containing the optimizer class and keyword arguments to use. Overrides `optim` and `optim_args` in `args`. Incompatible with the `optimizers` argument. Unlike `optimizers`, this argument avoids the need to place model parameters on the correct devices before initializing the Trainer. preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*): A function that preprocess the logits right before caching them at each evaluation step. Must take two tensors, the logits and the labels, and return the logits once processed as desired. The modifications made by this function will be reflected in the predictions received by `compute_metrics`. Note that the labels (second parameter) will be `None` if the dataset does not have them. Important attributes: - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`] subclass. - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`, the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`. - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from data parallelism, this means some of the model layers are split on different GPUs). - **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set to `False` if model parallel or deepspeed is used, or if the default `TrainingArguments.place_model_on_device` is overridden to return `False` . - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while in `train`) r )_get_learning_rate log_metricsmetrics_format save_metrics save_state tokenizerprocessing_classz5.0.0T)new_namerZraise_if_both_namesNNNrargs data_collator train_datasetzdatasets.Dataset eval_dataset model_initcompute_loss_funccompute_metrics callbacks optimizersoptimizer_cls_and_kwargspreprocess_logits_for_metricsc Cs |durd}td|dt|d}|jr)| dur)dt| jvr)td|j dur@|j dkr@|dur@td|j d |j t j ksI|j rR|jdurRtd ||_||_|jjrbt|jjnt|jjd|_d|_d |_||_|t|jj|_|j|}t ||j!|dur|dur||_"|#}nt$d |durt%&d t'||_"|j(j)t*vrtd|j(j)dt+|dd rt+|dd rd|_,nd |_,t+|dddurddt-|j./D}t0|dkrd|_,nt0|dkr|jj1t21|dk|_,nd |_,|j,rtd|jj3rNt4rJddl5m6}t7|t8r.||dn t9|drDt7|:t8rD||:dn t;dnt}t+|d dduoxt+|j=d!d }|rt9|d"rtd#|rt?|s|std$|r|std%|j=j@jAd&|j=j@jA|jBd'|_Ct0|jDdkr|jErtd(|jBd's|jFtGjHkrtd)|jI|_I|j,s|jEs|jJs|jKr|jLr|jCs|jMrd |_I|durt7|tNtOfrtP|ntQ}|dur|n||_R||_S||_T||_U|jIr0t+|d*dtVjWks0|X||j1|j,r8d|j_Y||_Z||_|j[\|}t?|sL|j]n|:j]}t|j}t9|d+rb|j^|__n t`d,d-|/D|__|ja|_a| |_b| |_c| \|_d|_e| |_f|jfdur|jddurt$d.|dur|jddus|jedurt$d/tgr|jddur|jD]}|j1}|jdjhD]}t0|d0dkr|d0dj1}nq||krtd1|jCs|jMr|jddus|jedurt$d2titj|jjk}| dur|n|| } tl| |j|jU|jd|je|_m|n|jjortpntqd |_rd|_s|jjtr1|u|jjvr?twjx|jjydd3tz|jRsStzt+|jRd4drStd5|j{dkrd|j|dkrdtd6|durxt}|sx|j{dkrxtd7|durt7|t2j~jjr|jrtd8d|_d |_d |_tr|jrtd9tr|jtjjjkrt;d:tjjjd;|jd|js|jr|jd?kr|j1t21d@kr|jrtstdAndB|_tdC|jdD|js|jr>|jEs>ts>|jdBkr-d|_t2j|_n|jdEkr>ts;t