o ZhY5@sddlZddlmZddlmZmZmZmZmZddl Z ddl m Z ddl m m ZddlmZddlmZddlmZmZmZddlmZdd lmZdd lmZdd lmZdd lm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&ddl'm(Z(m)Z)ddl*m+Z+ddl,m-Z-m.Z.m/Z/m0Z0m1Z1ddl2m3Z3m4Z4e0rddl5m6Z6ddl7m8Z8e19e:Z;Gddde j<Z=Gddde j<Z>Gddde j j<Z?Gddde j<Z@edGddde j<ZAGdd d e j<ZBd!e jCd"e jCd#e jCd$ee jCe jCffd%d&ZDd'e jCd(eEd$e jCfd)d*ZF +dcd,e jd>eLZMGd?d@d@ee-ZNGdAdBdBeLeZOeGdCdDdDe#ZPGdEdFdFe j j<ZQGdGdHdHe j<ZRdIdJZSGdKdLdLe j<ZTdMe jCd-e jCfdNdOZUd-e jCd.e jCdMe jCd$ee jCe jCffdPdQZVGdRdSdSe j<ZWGdTdUdUe j<ZXGdVdWdWe j<ZYGdXdYdYe j<ZZGdZd[d[e j<Z[Gd\d]d]e j<Z\Gd^d_d_eLZ]Gd`dadaeLeZ^gdbZ_dS)dN) dataclass)CallableListOptionalTupleUnion)Llama4VisionConfig)ACT2FN)Cache DynamicCacheHybridChunkedCache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPastCausalLMOutputWithPast ModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging) Llama4ConfigLlama4TextConfig) BlockMask)make_flex_block_causal_maskcs8eZdZdeffdd ZdejdejfddZZS)Llama4TextExpertsconfigcsxt|j|_|j|_|j|_|j|_tt |j|jd|j|_ tt |j|j|jf|_ t |j|_dSN)super__init__num_local_experts num_expertsintermediate_size hidden_sizeZ expert_dimnn Parametertorchempty gate_up_proj down_projr hidden_actact_fnselfr& __class__Y/var/www/auris/lib/python3.10/site-packages/transformers/models/llama4/modeling_llama4.pyr*1s  zLlama4TextExperts.__init__ hidden_statesreturncCs\||jd|j}t||j}|jddd\}}t||||j}|d|j}|S)a2 This should really not be run on a single machine, as we are reaching compute bound: - the inputs are expected to be "sorted" per expert already. - the weights are viewed with another dim, to match num_expert, 1, shape * num_tokens, shape Args: hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) selected_experts (torch.Tensor): (batch_size * token_num, top_k) routing_weights (torch.Tensor): (batch_size * token_num, top_k) Returns: torch.Tensor r(dim) viewr,r.r1Zbmmr3chunkr6r4)r8r=Zgate_upZgateupZ next_statesr;r;r<forward;s  zLlama4TextExperts.forward) __name__ __module__ __qualname__r"r*r1TensorrE __classcell__r;r;r9r<r%0s r%cs&eZdZdfdd ZddZZS) Llama4TextMLPNcsjt|dur |j}||_tj|j|dd|_tj|j|dd|_tj||jdd|_ t |j |_ dSNFbias) r)r*r-r&r/Linearr. gate_projup_projr4r r5 activation_fn)r8r&r-r9r;r<r*Rs zLlama4TextMLP.__init__cCs$|||||}||SN)rRrPrQr4)r8xr4r;r;r<rE^s zLlama4TextMLP.forwardrSrFrGrHr*rErJr;r;r9r<rKQs rKcs<eZdZd deffdd ZddZddZd d ZZS) Llama4TextL2Normư>epscst||_dSrS)r)r*rX)r8rXr9r;r<r*ds  zLlama4TextL2Norm.__init__cC$|t|djddd|jSNr(r?T)Zkeepdimr1ZrsqrtpowmeanrXr8rTr;r;r<_normh$zLlama4TextL2Norm._normcCs|||SrS)r_floattype_asr^r;r;r<rEkszLlama4TextL2Norm.forwardcCs d|jS)Nzeps=rXr8r;r;r< extra_reprn zLlama4TextL2Norm.extra_repr)rW) rFrGrHrar*r_rErerJr;r;r9r<rVcs rVcs6eZdZd fdd ZddZddZdd ZZS) Llama4TextRMSNormh㈵>cs&t||_tt||_dS)z< Llama4RMSNorm is equivalent to T5LayerNorm N)r)r*rXr/r0r1Zonesweight)r8r.rXr9r;r<r*ss zLlama4TextRMSNorm.__init__cCrYrZr[r^r;r;r<r_{r`zLlama4TextRMSNorm._normcCs|||}||jSrS)r_rarbri)r8rToutputr;r;r<rE~s zLlama4TextRMSNorm.forwardcCst|jjd|jS)Nz, eps=)tuplerishaperXrdr;r;r<reszLlama4TextRMSNorm.extra_repr)rh)rFrGrHr*r_rErerJr;r;r9r<rgrs rg Llama4TextMoec$eZdZfddZddZZS)rmcsPt|j|_|j|_|j|_t||_ t j |j|jdd|_ t ||_dSrL)r)r*Znum_experts_per_toktop_kr. hidden_dimr+r,r%expertsr/rOrouterrK shared_expertr7r9r;r<r*s  zLlama4TextMoe.__init__c Cs|j\}}}|d|j}||}||}tj||jdd\}}t|td d|| dd} tj ||j d dd| dd}t| |j} |ddd|}tj|d|d|j } | | dd} || } ||} | jd|| d|d| | fS) Nr?r r@z-infrdevice)inputrAindex)rArwsrc)rlreshaperprrr1ZtopkroZ full_likeraZscatter_ transposearangerurBexpandsizeZsigmoidtodtypegatherrqrsZ scatter_add_) r8r=batchZseq_lenrp router_logitsZtokens_per_expertZrouter_top_valueZrouter_indicesZ router_scoresZ routed_inZ routed_outoutr;r;r<rEs.   $  zLlama4TextMoe.forwardrUr;r;r9r<rms  cs8eZdZddeffdd ZeeddZZ S)Llama4TextRotaryEmbeddingNr&cspt|jdur dnd|_|j|_|j|_||_t|j|_ | |j|\}|_ |j d|dd|j |_ dS)NZllama3defaultinv_freqF) persistent)r)r*Z rope_scalingZ rope_typeZmax_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenr&rZ rope_init_fnattention_scalingZregister_bufferrZoriginal_inv_freq)r8r&rurr9r;r<r*s   z"Llama4TextRotaryEmbedding.__init__cCs|jddddf|jddd}|dddddf}t|jjtr2|jjdkr2|jjnd}tj |dd#| |j| dd}t t ||}||j}Wd|S1sawY|S) Nrr?r ZmpscpuF) device_typeenabledr()rrar|rl isinstancerutypestrr1Zautocastr~rzZpolarZ ones_liker)r8rT position_idsZinv_freq_expandedZposition_ids_expandedrfreqs freqs_cisr;r;r<rEs(&  z!Llama4TextRotaryEmbedding.forwardrS) rFrGrHr"r*r1Zno_gradrrErJr;r;r9r<rs rxqxkrr>c Cst|jg|jddddR}t|jg|jddddR}t||dddddddfd}t||dddddddfd}||||fS)Nr?r(r )r1view_as_complexraryrl view_as_realflattenrb)rrrZxq_Zxk_Zxq_outZxk_outr;r;r<apply_rotary_embs ,,,,rr=n_repcCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) r N)rlr|ry)r=rrnum_key_value_headsslenhead_dimr;r;r< repeat_kvs 0rmodulequerykeyvalueattention_maskscalingdropoutc Kst||j}t||j} t||dd|} |dur3|ddddddd|jdf} | | } tjj| dd} tjj | ||j d} t| | } | dd } | | fS)Nr(r r?r@ptrainingr ) rnum_key_value_groupsr1matmulrzrlr/ functionalsoftmaxrr contiguous rrrrrrrkwargs key_states value_states attn_weights causal_mask attn_outputr;r;r<eager_attention_forwards & rc Kst||j}t||j} t||dd|jd} |dur6|ddddddd|jdf} | | } tjj | dd} tjj | ||j d} t| | } | dd } | | fS) Nr(r rr?r@rr ) rrr1rrzrrlr/rrrrrrr;r;r<vision_eager_attention_forwards & rcseZdZdZdeffdd Z  ddejdeejejfde ejd e e d e ej d e e d eeje eje eejffd dZZS)Llama4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr&cs&t||_||_t|d|j|j|_|j|_|j|j|_ |j|_|jd|_ |j |_ |j |_ |j |_ |j|_d|_|j||_tj|j|j|j|jd|_tj|j|j|j|jd|_tj|j|j|j|jd|_tj|j|j|j|jd|_|jjr|jrt|j|_dSdSdS)NrrTrM)r)r*r& layer_idxgetattrr.num_attention_headsrrrr attn_scale floor_scaleattn_temperature_tuningattention_dropout is_causalno_rope_layersuse_roper/rOZattention_biasq_projk_projv_projo_projZ use_qk_normrV rms_norm_epsqk_normr8r&rr9r;r<r*#s:   zLlama4TextAttention.__init__Nr=position_embeddingsrpast_key_valuecache_positionrr>cKs|jdd}g|d|jR}|||} ||jg|d|jR} |||dd} |jrDt| | | | j \} } t |drS| | } | | } |j r|jstt|d|jd|jd} | d|dddfg|ddR} | |  | j} | dd} | dd} |durd|i} || | |j| \} } t}|jjdkr|jjdkr|d d rtd nt|jj}||| | | |f|jsd n|j |j!d |\}}|j"g|dR#}|$|}||fS)Nr?r r(r?reagersdpaoutput_attentionsF`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r)rr)%rlrrrBrrrzrrr~ruhasattrrrr1logfloorrarrr|rupdaterrr&_attn_implementationgetlogger warning_oncerrrrryrr)r8r=rrrrr input_shape hidden_shape query_statesrrZ attn_scalesZ cache_kwargsattention_interfacerrr;r;r<rEAsX      **      zLlama4TextAttention.forwardNN)rFrGrH__doc__r"r*r1rIrrr LongTensorrrrErJr;r;r9r<r s(#rcseZdZfddZ         ddejdeejdeejdeejd eeejd ee d ee d ee d eejdeeejejfde e deej eeej ej fffddZ ZS)Llama4TextDecoderLayercst|j|_t|||_|jduot|j||_||j v|_ |j r+t ||_ nt ||jd|_ t|j|jd|_t|j|jd|_||_dS)N)r-rc)r)r*r.r self_attnattention_chunk_sizeboolruse_chunked_attentionZ moe_layers is_moe_layerrm feed_forwardrKZintermediate_size_mlprgrinput_layernormpost_attention_layernormrrr9r;r<r*s     zLlama4TextDecoderLayer.__init__NFr=rchunk_causal_maskrrroutput_router_logits use_cacherrrr>c  Ks|} ||}|jr|dur|}|jd|| ||||| d| \} }| | }|} ||}||}|jr:|\}}nd}| || j}|f}|rN||f7}|rU||f7}|S)N)r=rrrrrrr;)rrrrrrrBrl)r8r=rrrrrrrrrrresidualZattention_statesZself_attn_weightsroutputsr;r;r<rEs:        zLlama4TextDecoderLayer.forward) NNNNFFFNN)rFrGrHr*r1rIrrrrrr FloatTensorrErJr;r;r9r<rsH       rc@s>eZdZeZdZdgZdZdZdZ dZ dZ dZ dZ ddZdS)Llama4PreTrainedModelTpast_key_valuesFcCsJt|jdr |jjn|jjj}t|tjr-|jjj d|d|j dur+|j j dSdSt|tj rN|jjj d|d|j durL|jj|j  dSdSt|tjrc|jjd|j j dSt|trq|jjddSt|tr|jjj d|d|jjj d|ddSt|tr|jjj |jd|jjj |jddSdS)Ninitializer_ranger)r]stdr)r)rr&r text_configrr/rOridataZnormal_rNZzero_ Embedding padding_idx LayerNormZfill_rgr%r3r4Llama4VisionModelclass_embeddingscalepositional_embedding_vlm)r8rrr;r;r< _init_weightss4          z#Llama4PreTrainedModel._init_weightsN)rFrGrHr! config_classZsupports_gradient_checkpointingZ_skip_keys_device_placementZ_supports_flash_attn_2Z_supports_sdpaZ_supports_flex_attnZ_supports_cache_classZ_supports_quantized_cacheZ_supports_static_cacheZ_supports_attention_backendrr;r;r;r<rs rcsZeZdZdgZdZeZdeffdd ZddZdd Z e e d+d e j d ee jd ee j deedee jdeedeedeedeedee j deedeeeffddZe jjdd  d,d e jde jde jdedef ddZded ed!ed"e jde jf d#d$Zed e jd%ed&ed'e j de jd(ef d)d*Z!Z"S)-Llama4TextModelrmodelr&cstj|_j|_tjj|j|_t fddt j D|_ t jjd|_td|_d|_|dS)Ncsg|]}t|qSr;)r).0rr&r;r< sz,Llama4TextModel.__init__..rcrF)r)r* pad_token_idr vocab_sizer/rr. embed_tokens ModuleListrangenum_hidden_layerslayersrgrnormr rotary_embgradient_checkpointing post_initr7r9rr<r*s   zLlama4TextModel.__init__cC|jSrSrrdr;r;r<get_input_embeddings z$Llama4TextModel.get_input_embeddingscC ||_dSrSrr8rr;r;r<set_input_embeddings  z$Llama4TextModel.set_input_embeddingsN input_idsrrr inputs_embedsrroutput_hidden_states return_dictrflash_attn_kwargsr>c Ks2|dur|n|jj}|dur|n|jj}|dur|n|jj}| dur$| n|jj} |du|duAr4td|jrC|jrC|rCt dd}|durR| | |j j j }|rq|durq|jjdurnt|j|jd|jd}nt}| dur|dur}|nd} tj| | |jd|j d} |dur| d}|j||| |||d\} }|}|||}|rdnd}|rdnd}|jd|jjD]>}|r||f7}|jr|jr||j|| ||||d|| | }n||f| |||||| |d | }|d}|r||df7}q||}|r ||f7}t||r|nd||d S) N:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Frr rt)rr;)rrrrrrrr)last_hidden_staterr= attentions)r&rrruse_return_dict ValueErrorr rrrrr~riruZget_text_configrr rlr Zget_seq_lengthr1r{ unsqueeze_update_causal_maskr r r _gradient_checkpointing_func__call__r r)r8rrrrrrrrrrrZpast_seen_tokensrrr=freq_cisZall_hidden_statesZall_self_attnsZ decoder_layer layer_outputsr;r;r<rEs              zLlama4TextModel.forwardF) recursiveT input_tensorc CsP|jjdkr|dur|dkr||fSdS|jjdvrdS|jd}||j}|jj} | du} |d} |dur@|p>|} n |durI|jdn|} | rq| | k} | | k| || k@}|rot | | |dt || || n| }|jjdkrt |tj r| r| t | | ddf}t || |||d }t ||| | dfd }||fSt |tr||fS|j|j}}| rt | | n| }|j||||||jdd }| rW| | krWt | | dd}||}|j|jj|||d }|dd||f}|jd| k}|r tj|d| |jdf}|s|dd| dddf}n |dd|ddf}||jdddd}||ddddddf}|jjd krWt|j}t |dk|d|}|jjdkr|dur|jjdvr|jdkr|st|j}t||}|jjdkr|dur|}|t|jk}tj||| |jdrd}||fS)NZflash_attention_2rr)rflex_attentionrr rr?r))offsets)Z query_length key_lengthr*)sequence_length target_lengthrr batch_size)startendrurr)cudaZxpuZnpu)rZpast_key_values_lengthZ is_training)r&ranyrlr~rurZget_max_cache_shaper1whererrImaxr$r#r5_prepare_4d_causal_attention_mask_with_cache_positioncreate_chunked_attention_maskr/rpadr|finfominrndimrZ_unmask_unattendedrZ_ignore_causal_mask_sdpar)r8rr(rrrZchunked_attention_maskrr,rZusing_chunked_attentionZfirst_cache_positionZfull_cache_lengthZcond1Zcond2r+r*rrur-rZ start_idxZend_idxZlocal_attention_maskZrequires_padding min_dtyper;r;r<r"s               z#Llama4TextModel._update_causal_maskrr/r0ruc Cs`tj|||d}t|d||d|}|d|d}|dk|dk@}||S)u Generate the following: 'What' : 0 ■ ⬚ ⬚ ⬚ ⬚ ⬚ | '▁is' : 1 ■ ■ ⬚ ⬚ ⬚ ⬚ | '▁ch' : 2 ■ ■ ■ ⬚ ⬚ ⬚ | 'unked' : 3 ⬚ ⬚ ⬚ ■ ⬚ ⬚ | '▁attention': 4 ⬚ ⬚ ⬚ ■ ■ ⬚ | '?' : 5 ⬚ ⬚ ⬚ ■ ■ ■ | If the chunk size is 3. This can just be applied over the already created attention mask rtrr )r1r{absr!r~) r8rr/r0ruZ arange_vectorZ block_posZ token_posmaskr;r;r<r7s z-Llama4TextModel.create_chunked_attention_maskr,r-rr.c KsD|dur|dkr|}|St|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9}|ddddddf |ddd}|dur| }|j d} |ddddddd| f|ddddddf |j} | dk} |ddddddd| f | ||ddddddd| f<|S) a Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. device (`torch.device`): The device to place the 4D attention mask on. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. Nr2Z fill_valuerrur Zdiagonalrtr?rrAr1r9r:fullruZtriur{ryr|clonerlr~ masked_fill rr,r-rrr.rrr<Z mask_lengthZ padding_maskr;r;r<r6s,   $ 6  zELlama4TextModel._prepare_4d_causal_attention_mask_with_cache_position) NNNNNNNNNN)FNT)#rFrGrH_no_split_modulesbase_model_prefixr"rr*rrrrr1rrrIr rrrrrrrrEcompilerdisabler"intrur7 staticmethodrr6rJr;r;r9r<rs      o  rc@s eZdZdS)KwargsForCausalLMN)rFrGrHr;r;r;r<rLTs rLc!s*eZdZdgZdZdgZddiZeZdeffdd Z d d Z d d Z d dZ ddZ ddZddZee            d'dejdeejdeejdeeeeejfdeejdeejdeedeedeed eed!eejd"eeejfd#eed$eeeffd%d&Z Z!S)(Llama4ForCausalLMrlanguage_modelzlm_head.weightlm_headZ colwise_repr&cs@t|t||_|j|_tj|j|jdd|_| dSrL) r)r*rrrr/rOr.rOrr7r9r;r<r*^s   zLlama4ForCausalLM.__init__cCs|jjSrSrrrdr;r;r<rgsz&Llama4ForCausalLM.get_input_embeddingscCs ||j_dSrSrPrr;r;r<rjrfz&Llama4ForCausalLM.set_input_embeddingscCrrSrOrdr;r;r<get_output_embeddingsmrz'Llama4ForCausalLM.get_output_embeddingscCrrSrQr8Znew_embeddingsr;r;r<set_output_embeddingsprz'Llama4ForCausalLM.set_output_embeddingscCrrSrr8decoderr;r;r< set_decodersrzLlama4ForCausalLM.set_decodercCrrSrUrdr;r;r< get_decodervrzLlama4ForCausalLM.get_decoderNrrrrrrlabelsrrrrrlogits_to_keeprr>c  Ks|dur|n|jj}| dur| n|jj} | dur| n|jj} |jd|||||||| d| d | }|d}t| tr@t| dn| }||dd|ddf}d}|durd|j d|||jj d| }t |||j |j |jdS)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, Llama4ForCausalLM >>> model = Llama4ForCausalLM.from_pretrained("meta-llama4/Llama4-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama4/Llama4-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```NT) rrrrrrrrrrr)logitsrZr)lossr\rr=rr;)r&rrrrrrJslicerOZ loss_functionrrrr=r)r8rrrrrrZrrrrrr[rrr=Z slice_indicesr\r]r;r;r<rEys>( zLlama4ForCausalLM.forward) NNNNNNNNNNNr)"rFrGrHrFrGZ_tied_weights_keys_tp_planr"rr*rrrRrTrXrYrrr1rrrIrr rrrrJrrLrrrErJr;r;r9r<rMWsp       rMc@seZdZUdZdZeejed<dZ ejed<dZ ee ejed<dZ ee ejed<dZee ejed<dZeejed<dS) Llama4CausalLMOutputWithPasta Base class for Llava causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size (batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nr]r\rr=rimage_hidden_states)rFrGrHrr]rr1r__annotations__r\rrr=rrrar;r;r;r<r`s r`crn)Llama4VisionMLP2cs\t|j|_|j|_tj|j|jdd|_tj|j|jdd|_ t |_ |j |_ dSrL)r)r*r.r-r/rOprojector_input_dimfc1projector_output_dimfc2GELUrRZprojector_dropoutrr7r9r;r<r*s   zLlama4VisionMLP2.__init__cCs8||}||}tj||j|jd}|||S)Nr)rerRFrrrgr8r=r;r;r<rEs  zLlama4VisionMLP2.forwardrUr;r;r9r<rcs  rccrn)Llama4MultiModalProjectorcs(ttj|jj|jjdd|_dSrL) r)r*r/rO vision_configZvision_output_dimrr.linear_1r7r9r;r<r*s  z"Llama4MultiModalProjector.__init__cCs||}|SrS)rm)r8image_featuresr=r;r;r<rE s z!Llama4MultiModalProjector.forwardrUr;r;r9r<rks rkc Cs|j\}}}tt|}||||d}|\}}}}|||t||t||}|dddd}||t||t||t||d}|dddd}||d|jd} | S)Nr?rr(r r )rlrJmathsqrtrBr}permuter) r(Z shuffle_ratior. num_patchesZchannels patch_sizeheightwidthZreshaped_tensorZ output_tensorr;r;r< pixel_shuffles  $rvc2eZdZfddZdejdejfddZZS)Llama4VisionPixelShuffleMLPcs>t|j|_t|j|jd|_|j|_t||_ dSr') r)r*pixel_shuffle_ratiorJrdZ inner_dimrfZ output_dimrcmlpr7r9r;r<r*#s z$Llama4VisionPixelShuffleMLP.__init__encoded_patchesr>cCst||j}||SrS)rvryrz)r8r{r;r;r<rE*s  z#Llama4VisionPixelShuffleMLP.forwardrFrGrHr*r1rIrErJr;r;r9r<rx" rxfreqs_cics(|jfddt|jD}|j|S)Ncs,g|]\}}|dks|dkr|ndqS)r r;)ridr;r;r<r2s,z)reshape_for_broadcast..)r; enumeraterlrB)r~rrlr;rr<reshape_for_broadcast0s rcCst|jg|jddddR}t|jg|jddddR}t||d}||j}t|| d}t|| d}| || |fS)Nr?r()r~rr ) r1rraryrlrr~rurrrb)rrr~Zquery_key_Z query_outZkey_outr;r;r<vision_apply_rotary_emb6s,,  rcsxeZdZdeffdd Z  d dejdejdeejdeed e e d e ejeejee ejff d d Z Z S)Llama4VisionAttentionr&cst||_|j|_|j|_|j|j|_d|_|j |_ |jd|_ t j |j|j|jdd|_ t j |j|j|jdd|_t j |j|j|jdd|_t j |j|j|jdd|_dS)Nr rTrM)r)r*r&r.Z embed_dimrZ num_headsrrrrr/rOrrrrr7r9r;r<r*Es   zLlama4VisionAttention.__init__Nr=r~rrrr>cKs"|jdd}g|d|jR}|||}|||} |||} t|| |d\}} |dd}| dd} | dd} t} |j j dvrd|j j dkr^| ddr^t d nt|j j } | ||| | df|jspd n|jddd |\} } | jg|dR} || } | | fS) Nr?)r~r r()rr)rrFrr)rrr)rlrrrBrrrrzrr&rrrrrrrryrr)r8r=r~rrrrrrrrrrrr;r;r<rETs@       zLlama4VisionAttention.forwardr)rFrGrHrr*r1rIrr rrrrErJr;r;r9r<rDs"rcrw)Llama4VisionMLPcsJt||_t|_tj|j|jdd|_ tj|j|jdd|_ dS)NTrM) r)r*r&r/rhrRrOr.r-rergr7r9r;r<r*s  zLlama4VisionMLP.__init__r=r>cCs"||}||}||}|SrS)rerRrgrjr;r;r<rEs   zLlama4VisionMLP.forwardr|r;r;r9r<rr}rc sPeZdZdeffdd Z  d dejdejdeejdeefd d Z Z S) Llama4VisionEncoderLayerr&csFt|j|_t||_t||_t|j|_ t|j|_ dSrS) r)r*r.rrrrzr/rrrr7r9r;r<r*s   z!Llama4VisionEncoderLayer.__init__N hidden_stater~rrcCsb|}||}|j|||d\}}||}|}||}||}||}|f}|r/||f7}|S)N)r~r)rrrrz)r8rr~rrrrrr;r;r<rEs      z Llama4VisionEncoderLayer.forwardr) rFrGrHrr*r1rIrrrErJr;r;r9r<rsrcsteZdZdZdeffdd Z    ddejdejdeejd ee d ee d ee d e e e ffd dZ ZS)Llama4VisionEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`Llama4VisionEncoderLayer`]. Args: config: Llama4VisionConfig r&cs@t|_tfddtjD|_d|_|_dS)Ncsg|]}tqSr;)r)r_rr;r<rsz0Llama4VisionEncoder.__init__..F) r)r*r&r/rrr r r r7r9rr<r*s   zLlama4VisionEncoder.__init__Nr=r~rrrrr>c Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|r"dnd}|r(dnd}|jD]/} |r6||f}|jrG|jrG|| j||||} n| ||||d} |rX|| df}| d}q-|rd||f}|srt dd|||fDSt |||dS) ad Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr;)rrrr~r rcs|] }|dur|VqdSrSr;rvr;r;r< z.Llama4VisionEncoder.forward..rr=r) r&rrrr r rr#r$rkr) r8r=r~rrrrZencoder_statesZall_attentionsZ encoder_layerr&r;r;r<rEsB       zLlama4VisionEncoder.forwardNNNN)rFrGrHrrr*r1rIrrrrrrErJr;r;r9r<rs,  rcrw)Llama4UnfoldConvolutioncs`t|j}t|tr||f}tjj||jd|_tj |j |d|d|j dd|_ dS)N) kernel_sizeZstriderr FrM) r)r*rsrrJr1r/ZUnfoldunfoldrO num_channelsr.linear)r8r&rr9r;r<r*s   z Llama4UnfoldConvolution.__init__r=r>cCs&||}|ddd}||}|S)Nrr(r )rrqrrjr;r;r<rE's  zLlama4UnfoldConvolution.forwardr|r;r;r9r<rs  rcrn)Llama4VisionRotaryEmbeddingc sdt|j|j}tj|dtjd|dd}tj||ddgdd}d|d<||}||}|j |j d}d|j td|dd|d |}|dd |ddddfj dd d}|dd |ddddfj dd d} tj|| gd d d dddf} | |d dddkd} ttjt| t| gd d} | |_dS) Nr()rr rr@r)r?r?r).Nr?.)r)r* image_sizersr1r{Zint32rycatr.rZ rope_thetaraZrepeat_interleaverrDrstackcossinr~) r8r&idxZimg_idxZ frequencies_xZ frequencies_yZfreq_dimZ rope_freqZfreqs_xZfreqs_yrr%r9r;r<r*/s   ,((($ z$Llama4VisionRotaryEmbedding.__init__cCs|j|jSrS)r~r~rurjr;r;r<rE@sz#Llama4VisionRotaryEmbedding.forwardrUr;r;r9r<r.s rcseZdZdZdgZeZdeffdd ZddZ    dd e j d e e j d e e d e e d e e de eee j dfff ddZZS)r vision_modelrr&cst||j|_|j|_|j|_|j|_|j|jdd|_|jd|_t||_ t |jt |j|_t |jt |j|j|_t||_t |j|_t |j|_t||_t||_|dS)Nr(r r)r)r*rrsr.rrrrrpatch_embeddingr/r0r1Zrandnrrrrotary_embeddingr layernorm_prelayernorm_postrrrxvision_adapterrr7r9r;r<r*Is       zLlama4VisionModel.__init__cCr)zg This function is used to fetch the first embedding layer to activate grads on inputs. )rrdr;r;r<rbsz&Llama4VisionModel.get_input_embeddingsN pixel_valuesrrrrr>.cCs|dur|n|jj}|dur|n|jj}|dur|n|jj}|j\}}}} d} d} ||} | j\} }}| || | ||} |j| jdd| jd}t j | |gdd} |d7}| || | ||} |j j | j | jd}| |} || } | |d|} ||}|j| d|||d}|j} || } | ddddddf} || } |r|jnd}|r|d}nd}|std d | ||fDSt| ||d S) a Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, MllamaVisionModel >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" >>> model = MllamaVisionModel.from_pretrained(checkpoint) >>> processor = AutoProcessor.from_pretrained(checkpoint) >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> output = model(**inputs) >>> print(output.last_hidden_state.shape) torch.Size([1, 1, 4, 1025, 7680]) ``` Nr rr?r@)rru)rrrr~r(csrrSr;rr;r;r<rrz,Llama4VisionModel.forward..r)r&rrrrlrryrr|r1rrr~rrurrBrrrrrr=rkr)r8rrrrrZbatch_size_times_num_tilesrrtruZnum_concurrent_mediaZ num_chunksrrrrrprZpositional_embeddingr~rjr=rr;r;r<rEhsZ        zLlama4VisionModel.forwardr)rFrGrHrGrFrrr*rr1rIrrrrrrErJr;r;r9r<rDs. rc(seZdZddgZiZdZeZdeffdd ZddZ d d Z d d Z d dZ ddZ ddZdejdeeeefdefddZe                d3dejdejdeejdeejdeeejdeejdeeeeefdeedeejd eed!eed"eed#eed$eejd%eeejfd&ejd'eed(eeeff$d)d*Z      d4d+d,Z e!dejd-ed.ed/ej"d$ejd0ef d1d2Z#Z$S)5Llama4ForConditionalGenerationrrr&cs^t|t|j|_t||_t|j|_ |jj |_ |j j dur&|j j nd|_ | dS)Nr?)r)r*rrlrrkmulti_modal_projectorrMrrNrr&rrr7r9r;r<r*s      z'Llama4ForConditionalGeneration.__init__cC |jSrS)rNrrdr;r;r<rrz3Llama4ForConditionalGeneration.get_input_embeddingscC|j|dSrS)rNrrr;r;r<rz3Llama4ForConditionalGeneration.set_input_embeddingscCrrS)rNrRrdr;r;r<rRrz4Llama4ForConditionalGeneration.get_output_embeddingscCrrS)rNrTrSr;r;r<rTrz4Llama4ForConditionalGeneration.set_output_embeddingscCrrS)rNrXrVr;r;r<rXrz*Llama4ForConditionalGeneration.set_decodercCrrS)rNrYrdr;r;r<rYrz*Llama4ForConditionalGeneration.get_decoderrvision_feature_layervision_feature_select_strategycKsJ|dvr td|jdd|D}|j|fddi|}|j}|S)a Obtains image last hidden states from the vision tower and apply al projection. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. vision_feature_layer (`Union[int, List[int]]`): The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). )rrBz$Unexpected select feature strategy: cSsi|] \}}|dur||qSrSr;)rkrr;r;r< szELlama4ForConditionalGeneration.get_image_features..rF)r ritemsrr)r8rrrrZ image_outputsrr;r;r<get_image_featuress z1Llama4ForConditionalGeneration.get_image_featuresNrrrrrrrZrrrrrr[ image_sizesrr>c# Ks| dur| n|jj} | dur| n|jj} | dur| n|jj} |dur$|n|jjj}|dur/|n|jjj}|du|duAr@td|durL|durLtd|durV||}|dur|j ||||d}|j }| d| d}| |}||jjkd}||j}| d| d}|dd}|}|| dkrtd|d | d|dd| d}|||}| |}|jd||||| | | | ||d |}|d}d}| durO|dur!|dd|j d d  df|j}|d ddddf||jdk}| d d df|| jdk} n|d ddddf}| d d df} t}!|!| d| d| d|j}| sg|f|d d}"|dure|f|"S|"St|||j|j|j|dury|d Sdd S)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, LlavaForConditionalGeneration >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf") >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf") >>> prompt = "USER: \nWhat's the content of the image? ASSISTANT:" >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, text=prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(**inputs, max_new_tokens=15) >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "USER: \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed" ```NrzdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one)rrrrr?).rrzMismatch: final_mask wants z0 embeddings, but multi_modal_projector returned ) rrrrrrrrrr[r .)r]r\rr=rrar;)r&rrrrlrrr rrrlrBr}rZimage_token_idr!r~rurysumr|Zmasked_scatterrNrr/ZCrossEntropyLossr`rr=r)#r8rrrrrrrrrZrrrrrr[rrrnZoriginal_inputs_embeds_shapeZ vision_flatZprojected_vision_flatZspecial_image_maskZ final_maskZ final_mask_1dZnum_tokens_to_fillZ expanded_maskrr\r]Zshift_attention_maskZ shift_logitsZ shift_labelsZloss_fctrjr;r;r<rE s1        (*&  z&Llama4ForConditionalGeneration.forwardc Ks8|jj|f|||||d|} |ddkr|| d<| S)N)rrrrr[rr)rNprepare_inputs_for_generation) r8rrrrrrr[rZ model_inputsr;r;r<rs  zs            !,   " cI'ap' A,Z 1