fThY5SSKrSSKJr SSKJrJrJrJrJr SSK r SSK J r SSK J s J r SSKJr SSKJr SSKJrJrJr SSKJr SS KJr SS KJr SS KJr SS KJ r J!r!J"r"J#r# SS K$J%r%J&r& SSK'J(r(J)r) SSK*J+r+ SSK,J-r-J.r.J/r/J0r0J1r1 SSK2J3r3J4r4 \0"5(a SSK5J6r6 SSK7J8r8 \1Rr"\:5r;"SS\ Rx5r="SS\ Rx5r>"SS\ RRx5r?"SS\ Rx5r@\"S5"SS\ Rx55rA"SS \ Rx5rBS!\ RS"\ RS#\ RS$\\ R\ R44S%jrDS&\ RS'\ES$\ R4S(jrFS[S)\ RxS*\ RS+\ RS,\ RS-\\ RS.\GS/\G4S0jjrHS[S)\ RxS*\ RS+\ RS,\ RS-\\ RS.\GS/\G4S1jjrI"S2S3\ Rx5rJ"S4S5\ Rx5rK\."S6S7\)55rL\."S8S9\L55rM"S:S;\\-5rN"S<S=\L\5rO\"S>S?\#55rP"S@SA\ RRx5rQ"SBSC\ Rx5rRSDrS"SESF\ Rx5rTSG\ RS*\ R4SHjrUS*\ RS+\ RSG\ RS$\\ R\ R44SIjrV"SJSK\ Rx5rW"SLSM\ Rx5rX"SNSO\ Rx5rY"SPSQ\ Rx5rZ"SRSS\ Rx5r["STSU\ Rx5r\"SVSW\L5r]"SXSY\L\5r^/SZQr_g)\N) dataclass)CallableListOptionalTupleUnion)Llama4VisionConfig)ACT2FN)Cache DynamicCacheHybridChunkedCache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPastCausalLMOutputWithPast ModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging) Llama4ConfigLlama4TextConfig) BlockMask)make_flex_block_causal_maskcj^\rSrSrS\4U4SjjrS\RS\R4SjrSr U=r $)Llama4TextExperts0configc>[TU]5 URUlURUlUR UlURUl[R"[R"URUR SUR -55Ul [R"[R"URUR UR 455Ul [URUlgN)super__init__num_local_experts num_expertsintermediate_size hidden_size expert_dimnn Parametertorchempty gate_up_proj down_projr hidden_actact_fnselfr) __class__s b/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/llama4/modeling_llama4.pyr.Llama4TextExperts.__init__1s !33!'!9!9!--00LLT5E5EtGWGWYZ]a]l]lYl)mnekk43C3CT__VZVfVf2g&hiV../ hidden_statesreturncTURURSUR5n[R"XR 5nUR SSS9up4[R"X@RU5-UR5nURSUR5nU$)a This should really not be run on a single machine, as we are reaching compute bound: - the inputs are expected to be "sorted" per expert already. - the weights are viewed with another dim, to match num_expert, 1, shape * num_tokens, shape Args: hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) selected_experts (torch.Tensor): (batch_size * token_num, top_k) routing_weights (torch.Tensor): (batch_size * token_num, top_k) Returns: torch.Tensor r,dim) viewr0r2r6bmmr8chunkr;r9)r=rBgate_upgateup next_statess r?forwardLlama4TextExperts.forward;s&**4+;+;RAQAQR ))M+<+<====+iikk$&7!7$..I !&&r4+;+;< rA)r;r9r3r8r2r1r0) __name__ __module__ __qualname____firstlineno__r#r.r6TensorrO__static_attributes__ __classcell__r>s@r?r'r'0s00/0U\\ellrAr'c2^\rSrSrSU4SjjrSrSrU=r$) Llama4TextMLPQcX>[TU]5 Uc URnXl[R "UR USS9Ul[R "UR USS9Ul[R "X!R SS9Ul [URUl gNFbias) r-r.r1r)r4Linearr2 gate_projup_projr9r r: activation_fn)r=r)r1r>s r?r.Llama4TextMLP.__init__Rs   $ & 8 8  6#5#57HuUyy!3!35FUS #46H6HuU#F$5$56rAcURURU55URU5-nURU5$N)rcrarbr9)r=xr9s r?rOLlama4TextMLP.forward^s7&&t~~a'89DLLOK ~~i((rA)rcr)r9rarbrfrQrRrSrTr.rOrVrWrXs@r?rZrZQs 7))rArZcF^\rSrSrSS\4U4SjjjrSrSrSrSr U=r $) Llama4TextL2Normcepsc.>[TU]5 Xlgrf)r-r.rm)r=rmr>s r?r.Llama4TextL2Norm.__init__ds rAcU[R"URS5RSSS9UR-5-$Nr,rET)keepdimr6rsqrtpowmeanrmr=rgs r?_normLlama4TextL2Norm._normh45;;quuQx}}R}>IJJJrAc^URUR55RU5$rf)rxfloattype_asrws r?rOLlama4TextL2Norm.forwardks"zz!'')$,,Q//rAc SUR3$)Nzeps=rmr=s r? extra_reprLlama4TextL2Norm.extra_reprnsdhhZ  rAr)gư>) rQrRrSrTr|r.rxrOrrVrWrXs@r?rkrkcs)EK0!!rArkc>^\rSrSrSU4SjjrSrSrSrSrU=r $)Llama4TextRMSNormrc>[TU]5 X l[R"[ R "U55Ulg)z, Llama4RMSNorm is equivalent to T5LayerNorm N)r-r.rmr4r5r6onesweight)r=r2rmr>s r?r.Llama4TextRMSNorm.__init__ss. ll5::k#:; rAcU[R"URS5RSSS9UR-5-$rqrsrws r?rxLlama4TextRMSNorm._norm{rzrAczURUR55RU5nX R-$rf)rxr|r}r)r=rgoutputs r?rOLlama4TextRMSNorm.forward~s.AGGI&..q1 ##rAc^[URR5SUR3$)Nz, eps=)tuplershapermrs r?rLlama4TextRMSNorm.extra_reprs' ))*+6$((<) rQrRrSrTr.rxrOrrVrWrXs@r?rrrs<K$==rAr Llama4TextMoec.^\rSrSrU4SjrSrSrU=r$)rc&>[TU]5 URUlURUlUR Ul[U5Ul [R"URUR SS9Ul [U5Ulgr])r-r.num_experts_per_toktop_kr2 hidden_dimr/r0r'expertsr4r`routerrZ shared_expertr<s r?r.Llama4TextMoe.__init__sp //  ,,!33(0 ii 2 2F4L4LSXY *62rAcURup#nURSUR5nURU5nX#-n[R "XPR SS9upx[R"U[S55RSX5RSS5n [R"XaRS9RSS5RU RS5S5n[R "U R55R#UR$5n URSS5RSU5n[R&"USUS9R#UR5n XRSS5-n UR)U 5n UR+U5n U R-SXRSU5S9 X4$) NrEr!rFz-infrdevice)inputrGindex)rGrsrc)rreshaperrr6topkr full_liker|scatter_ transposearangerrHexpandsizesigmoidtodtypegatherrr scatter_add_) r=rBbatchseq_lenr router_logitstokens_per_expertrouter_top_valuerouter_indices router_scores routed_in routed_outouts r?rOLlama4TextMoe.forwards%2%8%8" %--b$//B  M2 !O+0::mZZUV+W( OOM5= 9 B B1n g q qrsuv w  LL*3G3G H M MaQS T [ [\i\n\nop\qsu v  m&9&9&;<?? @S@ST '//A6==b*MLL   "] ! ! "   5 5b! << \\),   / Qn//"j:YZ!!rA)rrr0rrrrirXs@r?rrs3""rAcl^\rSrSrSS\4U4Sjjjr\R"5\S55r Sr U=r $)Llama4TextRotaryEmbeddingr)cX>[TU]5 URbSOSUlURUlURUlXl[URUl URURU5uo0l URSUSS9 URUl g)Nllama3defaultinv_freqF) persistent)r-r. rope_scaling rope_typemax_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr)r rope_init_fnattention_scalingregister_bufferroriginal_inv_freq)r=r)rrr>s r?r."Llama4TextRotaryEmbedding.__init__s %+%8%8%D)"("@"@$*$B$B! /?+/+<+>@GG HZHZ[\H]_acde ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk ^^ U C&))!((36KKVVWXZ[\E EOOE$:EBI!$:$::ID  D C s A(D== E )rr)rrrrrrf) rQrRrSrTr#r.r6no_gradrrOrVrWrXs@r?rrs7 // / / ]]_  rArxqxkrrCc *[R"UR5R"/URSSQSPSP765n[R"UR5R"/URSSQSPSP765n[R "X2SS2SS2SSS24-5R S5n[R "XBSS2SS2SSS24-5R S5nURU5URU54$)NrEr,r )r6view_as_complexr|rr view_as_realflattenr})rrrxq_xk_xq_outxk_outs r?apply_rotary_embrs    2 2 IBHHSbM I2 Iq I JC    2 2 IBHHSbM I2 Iq I JC   1dA &> > ? G G JF   1dA &> > ? G G JF >>" v~~b1 11rArBn_repcURup#pEUS:XaU$USS2SS2SSS2SS24RX#XU5nURX#U-XE5$)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) r!N)rrr)rBrrnum_key_value_headsslenhead_dims r? repeat_kvrs_ 2?1D1D.E z!!Qa"23::5W\dlmM  e(CT TTrAmodulequerykeyvalueattention_maskscalingdropoutc [X R5n[X0R5n [R"XR SS55U-n Ub"USS2SS2SS2SUR S24n X-n [ RRU SS9n [ RRXURS9n [R"X5n U R SS5R5n X4$)Nr,r rErFptrainingr!) rnum_key_value_groupsr6matmulrrr4 functionalsoftmaxrr contiguous rrrrrrrkwargs key_states value_states attn_weights causal_mask attn_outputs r?eager_attention_forwardr s3 ; ;L==((6??([L,,|:K''1-88:K  $$rAc [X R5n[X0R5n [R"XR SS55UR S--n Ub"USS2SS2SS2SUR S24n X-n [RRU SS9n [RRXURS9n [R"X5n U R SS5R5n X4$) Nr,r rrErFrr!) rrr6rrrrr4rrrrrrs r?vision_eager_attention_forwardr s3 ; ;L==((6??([L,,|:K''1-88:K  $$rAcB^\rSrSrSrS\4U4SjjrSS\RS\ \R\R4S\ \RS\ \ S \ \RS \ \S \ \R\ \R\ \ \R44S jjrS rU=r$)Llama4TextAttentioni z=Multi-headed attention from 'Attention Is All You Need' paperr)c>[TU]5 XlX l[ USUR UR -5UlUR UlUR UR-Ul URUlURS-Ul URUl URUl URUl URUlSUlUR UUl[$R&"UR UR UR-UR(S9Ul[$R&"UR URUR-UR(S9Ul[$R&"UR URUR-UR(S9Ul[$R&"UR UR-UR UR(S9UlURR2(a-UR"(aR65Ulggg)Nrr Tr^)r-r.r) layer_idxgetattrr2num_attention_headsrrrr attn_scale floor_scaleattn_temperature_tuningattention_dropout is_causalno_rope_layersuse_roper4r`attention_biasq_projk_projv_projo_proj use_qk_normrk rms_norm_epsqk_normr=r)rr>s r?r.Llama4TextAttention.__init__#s  " F4F4F&JdJd4de #)#=#= $*$>$>&B\B\$\!#)#=#= }}d*  ++!--'-'E'E$!'!9!9--i8 ii    : :T]] JQWQfQf  ii    : :T]] JQWQfQf  ii    : :T]] JQWQfQf  ii  & & 68J8JQWQfQf   ;; " "t}}+F,?,?@DL(5 "rArBposition_embeddingsrpast_key_valuecache_positionrrCc (URSSn/UQSPURP7nURU5RU5n UR U5R"/UQSPURP76n UR U5RU5R SS5n UR(a'[XURU R55up[US5(a"URU 5n URU 5n UR(aUR(d[R"[R "UR#5S-UR$- 5S-5UR&-S-n U RSUSSS45R)/UQSPSP75n X-RU R*5n U R SS5n U R SS5n Ub#SU0n UR-XUR.U 5up[0nUR2R4S:wadUR2R4S:Xa-UR7S S 5(a[8R;S 5 O[<UR2R4nU"UU U U U4UR>(dS O UR@URBS .UD6unnURD"/UQSP76RG5nURIU5nUU4$)NrEr!r,r"?r'eagersdpaoutput_attentionsF`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rr)%rrrrHrrrrrrrhasattrr"rr6logfloorr|rrrrupdaterr r)_attn_implementationgetlogger warning_oncerrrrrrr)r=rBr%rr&r'r input_shape hidden_shape query_statesrr attn_scales cache_kwargsattention_interfacer rs r?rOLlama4TextAttention.forwardAs$))#2. 88b8$--8 {{=166|D [[/44UkU2Ut}}U {{=166|DNNqRST =='7*=*@*@ATAT*U( $L 4 # #<< 5Lj1J  ' '  %++~';';'='CtGWGW&WX[^^_bfbqbqqtww &**A{21+EFMMNbP[Nb]^Nb`aNbcK(6::<;M;MNL#--a3 ))!Q/  %,n=L'5'<'d>d##L '>dkk>^>^&_#$7      % $}}C$2H2HLL %  % ! \"));;;;FFH kk+. L((rA)rrrr)rrrrrrrrrrr"rrrNN)rQrRrSrT__doc__r#r.r6rUrrr LongTensorrrrOrVrWrXs@r?rr sGA/AF+/59 ?)||?)#5<<#=>?)!. ?) ! ?) !!1!12 ?)-.?) u||Xell3XeELL>Q5RR S?)?)rArc^\rSrSrU4SjrSS\R S\\R S\\R S\\RS\\ \R S\\ S \\ S \\ S \\RS \\ \R \R 4S \ \ S\ \R\\ \R\R444SjjrSrU=r$)Llama4TextDecoderLayeric>[TU]5 URUl[X5UlUR SL=(a [ URU5UlX!R;Ul UR(a[U5Ul O[XRS9Ul [URUR S9Ul[URUR S9UlX lg)N)r1r)r-r.r2r self_attnattention_chunk_sizeboolruse_chunked_attention moe_layers is_moe_layerr feed_forwardrZintermediate_size_mlprr!input_layernormpost_attention_layernormrr#s r?r.Llama4TextDecoderLayer.__init__s !--,V?%+%@%@%L%wQUV\VkVkluVvQw"%):)::    -f 5D  -fHdHd eD 01C1CI\I\](9&:L:LRXReRe(f%"rArBrchunk_causal_maskrr&r,output_router_logits use_cacher'r%rrCc Un URU5nUR(aUbUnUR"SUU UUUUU S.U D6upX-nUn URU5nUR U5nUR (aUupOSnXR U R5-nU4nU(aUU4- nU(aUU4- nU$)N)rBr%rr&r,rQr')rLrGrDrMrJrIrHr)r=rBrrOrr&r,rPrQr'r%rresidualattention_statesself_attn_weightsroutputss r?rOLlama4TextDecoderLayer.forwards!,,];   % %*;*G.N/3nn / ' 3))/) /  / +!3 !55mD ))-8   +8 (M= M #5#5hnn#EE  "  )+ +G   ' 'GrA)rJr2rLrIrrMrDrG) NNNNFFFNN)rQrRrSrTr.r6rUrr@rrFrr FloatTensorrOrVrWrXs@r?rBrBs3#&2648378<,1/4$)59KO5||5!.5$ELL1 5 u//0 5 !u||!45 5$D>5'tn5D>5!!1!125&eELL%,,,F&GH5-.5 u  (51B1BEDUDU1U+V"WW X55rArBcD\rSrSr\rSrS/rSrSr Sr Sr Sr Sr SrSrSrg)Llama4PreTrainedModeliTpast_key_valuesFc|[URS5(aURROURRRn[ U[ R 5(aWURRRSUS9 URb%URRR5 gg[ U[ R5(adURRRSUS9 URb2URRURR5 gg[ U[ R5(aJURRRS5 URRR5 g[ U[ 5(a&URRRS5 g[ U["5(aIUR$RRSUS9 UR&RRSUS9 g[ U[(5(a[UR*RRUR,S9 UR.RRUR,S9 gg)Ninitializer_ranger.)rvstdr))r_)r/r)r^ text_configrr4r`rdatanormal_r_zero_ Embedding padding_idx LayerNormfill_rr'r8r9Llama4VisionModelclass_embeddingscalepositional_embedding_vlm)r=rr_s r? _init_weights#Llama4PreTrainedModel._init_weightsst{{$788 KK ) )((:: fbii ( ( MM   & &CS & 9{{&   &&('  - - MM   & &CS & 9!!- ""6#5#56<<>.  - - MM   $ $S ) KK   " " $  1 2 2 MM   $ $S )  1 2 2    $ $ , ,#3 , ?    ! ! ) )s ) <  1 2 2  " " ' ' / /FLL / A  + + 0 0 8 8V\\ 8 J3rArSN)rQrRrSrTr" config_classsupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_flex_attn_supports_cache_class_supports_quantized_cache_supports_static_cache_supports_attention_backendrlrVrSrAr?r[r[sEL&*##4"5"N  $!"&KrAr[c^\rSrSrS/rSr\rS\4U4SjjrSr Sr \ \ S$S\ RS \\ R S \\ RS \\S \\ R$S \\S\\S\\S\\S\\ RS\\S\\\44Sjj55r\ R4R7SS9S%S \ R S\ R S\ R S \S\4 Sjj5rS\S\S\S\ R<S\ R 4 Sjr\ S \ R S\S\S \ RBS\ R S!\4 S"j5r"S#r#U=r$$)&Llama4TextModelirBmodelr)c >[TU]U5 URUlURUl[ R "URURUR5Ul[ R"[UR5Vs/sHn[X5PM sn5Ul [URURS9Ul[#US9UlSUlUR)5 gs snf)Nr)r)F)r-r. pad_token_idre vocab_sizer4rdr2 embed_tokens ModuleListrangenum_hidden_layersrBlayersrr!normr rotary_embgradient_checkpointing post_initr#s r?r.Llama4TextModel.__init__s  !.. ++LL):):FQ>QR 36B&+#  isC?cUR$rfr~rs r?get_input_embeddings$Llama4TextModel.get_input_embeddings s   rAcXlgrfrr=rs r?set_input_embeddings$Llama4TextModel.set_input_embeddings s!rA input_idsrrr\ inputs_embedsrQr,output_hidden_states return_dictr'flash_attn_kwargsrCc  UbUOURRnUbUOURRnUbUOURRnU bU OURRn USLUSL- (a [ S5eUR (a/UR(aU(a[RS5 SnUc>URURURRR55nU(adUcaURR5Rb2[!URUR"SUR"S5nO [%5nU cDUbUR'5OSn [(R*"XUR"S-URS9n UcU R-S5nUR/X%XXvS9upUnUR1X5nU(aSOSnU(aSOSnUR2SURR4HnU(aUU4- nUR (a7UR(a&UR7UR8UU UUUUSUU U5 nOU"U4U UUUUUU US .U D6nUSnU(dMxUUS4- nM UR;U5nU(aUU4- n[=UU(aUOSUUS 9$) N:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Frr!r)rQrS)rrOrr&r,rQr'r%)last_hidden_stater\rB attentions)r)r,rrQuse_return_dict ValueErrorrrr5r6r~rrrget_text_configrErrr get_seq_lengthr6r unsqueeze_update_causal_maskrrr_gradient_checkpointing_func__call__rr)r=rrrr\rrQr,rrr'rpast_seen_tokensrrOrBfreq_cisall_hidden_statesall_self_attns decoder_layer layer_outputss r?rOLlama4TextModel.forwards 2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKL]*B* & & ??=?#7BD0d![[)H4;;+H+HIM#!m%55!**t}} $ A A!**!% #%" ! !.! !#.&7!-#2&7'#1(0 !( ! *!,M  =#3"55GJJ -0  -!1 1 &+/8Od+%   rAF) recursive input_tensorc URRS:XaUbUS:HR5(aX4$gURRS;agURSnUR UR 5nURR n U SLn USn UbUR5=(d Un OUbURSOUn U (aNX:n X:X-U :-nU(a5[R"U X-S- [R"XU-U 55OU nURRS:Xan[U[R5(a7U (aU [X- S-S54n[XUWUS 9n[UUU U S4S 9nX4$[U[5(aX4$URUR nnU (a [X5OU nUR!UUUUUURSS 9nU (GaNX:GaH[X- S-S5nUW-nUR#URR UUUS 9nUSS2UU24nURSU :nU(a2[$R&R)USU URS- 45nU(dUSSU*S2SS24nO USSUSS24nUR+URSSSS5nUUSS2SSSS24-nURRS :XaJ[R,"U5R.n[R"US:HUS5R U5nURRS:XakUbhUR R0S;aNUR2S:Xa>U(d7[R,"U5R.n[4R6"UU5nURRS:Xa^Ub[UR95nU[R,"U5R.:gn[4R:"UUU UR<S9(aSnUU4$)Nflash_attention_2r.r>)r+flex_attentionr*r!rrEr)offsets) query_length key_lengthr)sequence_length target_lengthrr' batch_size)startendrr*r+)cudaxpunpu)rpast_key_values_length is_training)r)r3anyrrrrEget_max_cache_shaper6whererrUmaxr%r$r5_prepare_4d_causal_attention_mask_with_cache_positioncreate_chunked_attention_maskr4rpadrfinfominrndimr_unmask_unattendedrF_ignore_causal_mask_sdpar)r=rrr'r\r,chunked_attention_maskrQrrEusing_chunked_attentionfirst_cache_positionfull_cache_lengthcond1cond2rrrrrr start_idxend_idxlocal_attention_maskrequires_padding min_dtypes r?r#Llama4TextModel._update_causal_masks ;; + +/B B)~/D.I.I.K.K%55 ;; + +3V V&,,Q/'**4;;7#{{??"6d"B-a0  & / C C E X KKo'MOcd '  ;; + +/? ?.%,,77*3S9M9dgh9hjk5lmG-H&ozcj.*"="!00115 " &==.)44%==%**L,?,?vH_-Dev PP +')#))!, Q  #'8'O0G!KQOI*,G%)%G%G 00 &H& "$2!Yw5F2F#G 399"=@TT ')}}'8'8(1.BEYE_E_`bEc.c*d($$)?d_L\L]_`@`)a&)?dN\]@])^&%;%B%BRSTVZ\`bcSc>d%d "{{//7:!KK.22 ).5Kq5PR[]`)a)d)dej)k& KK , , 6*%%**.DD##q(%  E*..I0CCKQZ[K ;; + +v 5:P:\%;%@%@%B "%U);)?)??K%>>*'; MM  # 222rArErrrc$[R"X#US9n[R"URS5U-URS5U-- 5nURS5URS5- nUS:HUS:*-nUR U5$)u Generate the following: 'What' : 0 ■ ⬚ ⬚ ⬚ ⬚ ⬚ | '▁is' : 1 ■ ■ ⬚ ⬚ ⬚ ⬚ | '▁ch' : 2 ■ ■ ■ ⬚ ⬚ ⬚ | 'unked' : 3 ⬚ ⬚ ⬚ ■ ⬚ ⬚ | '▁attention': 4 ⬚ ⬚ ⬚ ■ ■ ⬚ | '?' : 5 ⬚ ⬚ ⬚ ■ ■ ■ | If the chunk size is 3. This can just be applied over the already created attention mask rrr!)r6rabsrr) r=rErrr arange_vector block_pos token_posmasks r?r-Llama4TextModel.create_chunked_attention_masks  U? II  # #A &*> >AXAXYZA[_sAs s "++A.1H1H1KK Q9>2wwvrArrrrc UbUR5S:XaUnU$[R"U5Rn[R"X4XUR S9nUS:wa[R "USS9nU[R"X$R S9URSS5:-nUSSSS2SS24RUSSS5nUbUR5nURSn USS2SS2SS2SU 24USS2SSSS24RUR 5-n U S:Hn USS2SS2SS2SU 24RX5USS2SS2SS2SU 24'U$) a Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. device (`torch.device`): The device to place the 4D attention mask on. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. Nr fill_valuerrr!diagonalrrErrGr6rrfullrtriurrrclonerr masked_fill rrrrr'rrrr mask_length padding_masks r?rELlama4TextModel._prepare_4d_causal_attention_mask_with_cache_positions~@  %.*<*<*>!*C(K*' E*..I** 0Y\j\q\qK!##jjqA 5<< >S>STWeWmWmnprsWtt tK%dD!Q&67>>z1bRTUK))//1 ,2226 *1aL[L+@ANSTVZ\`bcScDdDgDg"))E   ,q0 5@Aq,;,AV5W5c5c 6 Aq!\k\12rA)r~rrrrerr}) NNNNNNNNNN)FNT)%rQrRrSrT_no_split_modulesbase_model_prefixr#rnr.rrrrr6r@rrUr rYrFrrrrrrOcompilerdisablerintrr staticmethodrrrVrWrXs@r?ryrysF12#L/ !"'+1537+/59$(,0/3&*59m ##m !.m u//0 m "% m   1 12 m D>m $D>m 'tnm d^m !!1!12m $$89m  u-- .m m ^ ^^e,#(#~3 ~3ll~3 ~3  ~3  ~3-~3@$'03:=GL|| 06 666{{ 6  6  66rAryc\rSrSrSrg)KwargsForCausalLMiTrSN)rQrRrSrTrVrSrAr?rrTs3rArc!^\rSrSrS/rSrS/rSS0r\r S\4U4Sjjr S r S r S r S rS rSr\\SS\R(S\\R,S\\R(S\\\\\R44S\\R4S\\R(S\\S\\S\\S\\S\\R(S\\\R,4S\\S\\\ 44Sjj55r!Sr"U=r#$) Llama4ForCausalLMiWrBlanguage_modelzlm_head.weightlm_head colwise_repr)c>[TU]U5 [U5UlURUl[ R "URURSS9UlUR5 gr]) r-r.ryrzr}r4r`r2rrr<s r?r.Llama4ForCausalLM.__init__^sU  $V,  ++yy!3!3V5F5FUS  rAc.URR$rfrzr~rs r?r&Llama4ForCausalLM.get_input_embeddingsgszz&&&rAc$XRlgrfrrs r?r&Llama4ForCausalLM.set_input_embeddingsjs "' rAcUR$rfrrs r?get_output_embeddings'Llama4ForCausalLM.get_output_embeddingsms ||rAcXlgrfrr=new_embeddingss r?set_output_embeddings'Llama4ForCausalLM.set_output_embeddingsps% rAcXlgrfrzr=decoders r? set_decoderLlama4ForCausalLM.set_decoderss rAcUR$rfrrs r? get_decoderLlama4ForCausalLM.get_decodervs zzrArrrr\rlabelsrQr,rrr'logits_to_keeprrCc  UbUOURRnU bU OURRn U bU OURRn UR"SUUUUUUUU SU S. U D6nUSn[ U [ 5(a [U *S5OU nURUSS2USS245nSnUb)UR"SUX`RRS.U D6n[UUURURURS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, Llama4ForCausalLM >>> model = Llama4ForCausalLM.from_pretrained("meta-llama4/Llama4-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama4/Llama4-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```NT) rrrr\rrQr,rrr'r)logitsrr})lossrr\rBrrS)r)r,rrrzrrslicer loss_functionr}rr\rBr)r=rrrr\rrrQr,rrr'rrrWrB slice_indicesrrs r?rOLlama4ForCausalLM.forwardys1P2C1N-TXT_T_TqTq$8$D $++JjJj &1%SV8W8W~ot4]k mA}a,?@A  %%pVF{{OeOepiopD%#33!//))   rA)rrzr}) NNNNNNNNNNNr)$rQrRrSrTrr_tied_weights_keys_tp_planr#rnr.rrrrr r rrr6r@rrUrr rrYrFrrrrrrOrVrWrXs@r?rrWs12(*+=)H#L/'(&'+1537KO59-1$(,0/3&*5934I ##I !.I u//0 I "%tE4E4E/F(F"GH I   1 12 I ))*I D>I $D>I 'tnI d^I !!1!12I c5<</0I *+I  u,, -I I rArc\rSrSr%SrSr\\R\ S'Sr \R\ S'Sr \\ \R\ S'Sr \\\R\ S'Sr\\\R\ S'Sr\\R\ S 'S rg) Llama4CausalLMOutputWithPastia< Base class for Llava causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size (batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nrrr\rBrimage_hidden_statesrS)rQrRrSrTr?rrr6rY__annotations__rr\rrBrrrrVrSrAr?rrs<)-D(5$$ %, $FE  $9=OXd5#4#456=8[TU]5 URUlURUl[R "URUR SS9Ul[R "URURSS9Ul [R"5Ul URUl gr])r-r.r2r1r4r`projector_input_dimfc1projector_output_dimfc2GELUrcprojector_dropoutrr<s r?r.Llama4VisionMLP2.__init__s !--!'!9!999T33V5O5OV[\99V88&:U:U\abWWY// rAcURU5nURU5n[R"XRURS9nURUR U55$)Nr)r!rcFrrr#r=rBs r?rOLlama4VisionMLP2.forwardsR/ **=9  -<<$--X !!$((="9::rA)rcrr!r#r2r1rirXs@r?rrs0;;rArc.^\rSrSrU4SjrSrSrU=r$)Llama4MultiModalProjectoric>[TU]5 [R"URR UR RSS9Ulgr]) r-r.r4r` vision_configvision_output_dimr`r2linear_1r<s r?r."Llama4MultiModalProjector.__init__s?   2 2    * *  rAc(URU5nU$rfr0)r=image_featuresrBs r?rO!Llama4MultiModalProjector.forward s n5 rAr3rirXs@r?r,r,s rAr,c 8URup#n[[R"U55nUR X%US5nUR 5up&ptUR X&[Xq-5[XA- 55nUR SSSS5R5nUR U[Xa-5[Xq-5[XAS-- 55nUR SSSS5R5nUR USURS5n U $)NrErr,r!r )rrmathsqrtrHrpermuter) r shuffle_ratior num_patcheschannels patch_sizeheightwidthreshaped_tensor output_tensors r? pixel_shufflerBs(4(:(:%JXTYY{+,J$$ZZLL*6*;*;*='J"'' C@U[TU]5 URUl[URURS--5UlUR Ul[U5Ul gr+) r-r.pixel_shuffle_ratiorr  inner_dimr" output_dimrmlpr<s r?r.$Llama4VisionPixelShuffleMLP.__init__#sX #)#=#= V77D[R"UR5R"/URSSQSPSP765n[R"UR5R"/URSSQSPSP765n[ X#S9nUR UR5n[R"X2-5RS5n[R"XB-5RS5nURU5URU54$)NrEr,)rOrr ) r6rr|rrrTrrrrr})rrrOquery_key_ query_outkey_outs r?vision_apply_rotary_embrZ6s  " "5;;=#8#8#R%++cr:J#RB#RPQ#R SF  !4!4!Lciin!Lb!L!!L MD$hEH{{6==)H""6#45==a@I  199![TU]5 XlURUlUR UlURUR -UlSUlURUl URS-Ul [R"URUR UR-SS9Ul [R"URUR UR-SS9Ul[R"URUR UR-SS9Ul[R"UR UR-URSS9Ulg)Nr!r Tr^)r-r.r)r2 embed_dimr num_headsrrrrr4r`rrrrr<s r?r.Llama4VisionAttention.__init__Es  ++33**f.H.HH $%!!'!9!9}}d* ii0NUYZ ii0NUYZ ii0NUYZ ii >UYZ rArBrOrr&rrCc ^URSSn/UQSPURP7nURU5RU5nUR U5RU5n UR U5RU5n [ XUS9upURSS5nU RSS5n U RSS5n [n URRS;adURRS:Xa-URSS5(a[RS 5 O[URRn U "UUU U S4UR(dS O UR SSS .UD6upU R""/UQSP76R%5n UR'U 5n X4$) NrErOr!r,)r*rr+r,Fr-r.)rrr)rrrrHrrrZrr r)r3r4r5r6rrrrrr)r=rBrOrr&rr7r8r9rrr<r rs r?rOLlama4VisionAttention.forwardTs$))#2. 88b8$--8 {{=166|D [[/44\B {{=166|D #:<^f#g #--a3 ))!Q/ #--a3 (F ;; + +3N N{{//69fjjI\^c>d>d##L '>dkk>^>^&_#$7      % $}}C$2H2H %  % ! "));;;;FFH kk+. ((rA) rr)r^rrr_rrrrrr>)rQrRrSrTr r.r6rUrr rrrrOrVrWrXs@r?r\r\Ds [1 [&26*. .)||.),,.)!. .) ! .) -. .) u||Xell3XeELL>Q5RR S.).)rAr\cb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)Llama4VisionMLPic>[TU]5 Xl[R"5Ul[R "URURSS9Ul [R "URURSS9Ul g)NTr^) r-r.r)r4r$rcr`r2r1r!r#r<s r?r.Llama4VisionMLP.__init__sc  WWY99V//1I1IPTU99V55v7I7IPTUrArBrCclURU5nURU5nURU5nU$rf)r!rcr#r)s r?rOLlama4VisionMLP.forwards4/ **=9 / rA)rcr)r!r#rNrXs@r?reres)VU\\ellrArec ^\rSrSrS\4U4SjjrS S\RS\RS\\RS\\ 4Sjjr S r U=r $) Llama4VisionEncoderLayerir)c>[TU]5 URUl[U5Ul[ U5Ul[R"UR5Ul [R"UR5Ul grf) r-r.r2r\rDrerIr4rfrLrMr<s r?r.!Llama4VisionEncoderLayer.__init__sb !--.v6"6*!||F,>,>?(* V5G5G(H%rA hidden_staterOrr,cUnURU5nURUUUS9upXQ-nUnURU5nURU5nXQ-nU4nU(aXv4- nU$)N)rOr)rLrDrMrI)r=rnrOrr,rTrrWs r?rO Llama4VisionEncoderLayer.forwards ++L9 %)^^ )&4& "  .  44\B xx - . /   &GrA)r2rLrIrMrDr>) rQrRrSrTr r.r6rUrrFrOrVrWrXs@r?rkrks_I1I26,0 ll,,!.  $D> rArkc^\rSrSrSrS\4U4SjjrSS\RS\RS\ \RS\ \ S \ \ S \ \ S \ \ \ 44S jjrS rU=r$)Llama4VisionEncoderiz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`Llama4VisionEncoderLayer`]. Args: config: Llama4VisionConfig r)c>[TU]5 Xl[R"[ UR 5Vs/sHn[U5PM sn5UlSUl Xlgs snf)NF) r-r.r)r4rrrrkrr)r=r)_r>s r?r.Llama4VisionEncoder.__init__sY  mmuU[UmUmOn$oOn!%=f%EOn$op &+# %psA,rBrOrr,rrrCc.UbUOURRnUbUOURRnUbUOURRnU(aSOSnU(aSOSnURHnn U(aXq4-nUR (a1UR (a URU RUUUU5n O U "UUUUS9n U(aXS4-nU SnMp U(aXq4-nU(d[SXU455$[XUS9$)a Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NrS)rnrr,rOr!rc3.# UH ocMUv M g7frfrS.0vs r? .Llama4VisionEncoder.forward..se$Sq$S rrBr) r)r,rrrrrrrrr) r=rBrOrr,rrencoder_statesall_attentions encoder_layerrs r?rOLlama4VisionEncoder.forwards'>2C1N-TXT_T_TqTq$8$D $++JjJj &1%>Ne]N$See e+Vd  rA)r)rrNNNN)rQrRrSrTr?r r.r6rUrrFrrrrOrVrWrXs@r?rrrrs126,0/3&*G ||G ,,G !. G $D> G 'tn G d^G  uo% &G G rArrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)Llama4UnfoldConvolutionic8>[TU]5 URn[U[5(aX"4n[ R RX!RS9Ul[ R"URUS-US-URSS9Ul g)N) kernel_sizestriderr!Fr^) r-r.r=rrr6r4Unfoldunfoldr` num_channelsr2linear)r=r)rr>s r?r. Llama4UnfoldConvolution.__init__s '' k3 ' '&4Khhoo+FWFWoX ii   +a. 0;q> A     rArBrCcpURU5nURSSS5nURU5nU$)Nrr,r!)rr9rr)s r?rOLlama4UnfoldConvolution.forward's8 M2 %--aA6  M2 rA)rrrNrXs@r?rrs(  U\\ellrArc.^\rSrSrU4SjrSrSrU=r$)Llama4VisionRotaryEmbeddingi.c>[T U]5 URUR-n[R "US-[R S9RUS-S5n[R"X3SS/SS9nSUS'X2-nX2-nURUR-S-nSUR[R "SUS5SUS-R5U- -- nUS-S USSSS24-RSS S9nUS-S USSSS24-RSS S9n [R"X/S S9R5R5S SSS24n U RURS SS5S:S5n [R "[R""[R$"U 5[R&"U 5/S S95n Xlg) Nr,)rr!rrFr)rErEr)).NrE.)r-r. image_sizer=r6rint32rcatr2r rope_thetar|repeat_interleaverrrstackcossinrO) r=r)idximg_idx frequencies_x frequencies_yfreq_dim rope_freqfreqs_xfreqs_yrrr>s r?r.$Llama4VisionRotaryEmbedding.__init__/s 6#4#44,,sAvU[[9AA#q&!L))Wbqk2:  %%)C)CCqH6,,a11MN_QY]^Q^1`1f1f1hks1stu !A%y1IdD!m4LL__`agi_j!A%y1IdD!m4LL__`agi_j 7,"5;;=HHJ3PSRSPS8T!!'//"a";a"?C((eii6F RWHX5Y_a)bc rAcLURRUR5$rf)rOrrr)s r?rO#Llama4VisionRotaryEmbedding.forward@s}} 4 455rArbrirXs@r?rr.s!"66rArc^\rSrSrSrS/r\rS\4U4SjjrSr SS\ RS\ \ RS \ \ S \ \ S \ \ S \\\\ RS 444 SjjrSrU=r$)rhiD vision_modelrkr)c~>[TU]U5 URUlURUlURUlUR UlURUR-S-S-UlURS-Ul[U5Ul [R"UR[R"UR5-5Ul[R"UR[R"UR UR5-5Ul[!U5Ul[R$"UR5Ul[R$"UR5Ul[+U5Ul[/U5UlUR35 g)Nr,r!r )r-r.rr=r2rr;rjrpatch_embeddingr4r5r6randnrirkrrotary_embeddingrf layernorm_prelayernorm_postrrrzrDvision_adapterrr<s r?r.Llama4VisionModel.__init__IsA   ++ ++!--"// OOt>1DqH''- 6v>!||DJJTEUEU9V,VW(* TZZ%++dN^N^`d`p`pBq5q(r% ;F C \\$*:*:; ll4+;+;<)0 9&A rAcUR$)zW This function is used to fetch the first embedding layer to activate grads on inputs. )rrs r?r&Llama4VisionModel.get_input_embeddingsbs###rA pixel_valuesrr,rrrC.c UbUOURRnUbUOURRnUbUOURRnURupgpSn Sn UR U5n U RupnU R Xj-U -X5n URRU RSSU RS5n[R"U U/SS9n US- nU R Xj-XU5n URRU RU RS9nU U-n URU 5n U R!USU5n UR#U5nUR%U SUUUS9nUR&n UR)U 5n U SS2SS2SS24n UR+U 5n U(a UR,OSnU(aUSnOSnU(d[/S U UU455$[1U UUS 9$) aN Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, MllamaVisionModel >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" >>> model = MllamaVisionModel.from_pretrained(checkpoint) >>> processor = AutoProcessor.from_pretrained(checkpoint) >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> output = model(**inputs) >>> print(output.last_hidden_state.shape) torch.Size([1, 1, 4, 1025, 7680]) ``` Nr!rrErF)rr)rrr,rOr,c3.# UH ocMUv M g7frfrSrxs r?r{,Llama4VisionModel.forward..s_$Mq$Mr}r~)r)r,rrrrrrirr6rrkrrrrrHrrzrrrrBrr)r=rrr,rrbatch_size_times_num_tilesrr>r?num_concurrent_media num_chunksrnrtr;rripositional_embeddingrOrrBrs r?rOLlama4VisionModel.forwardhsB>2C1N-TXT_T_TqTq$8$D $++JjJj &1% _ 'tn _ d^ _  ellC&7 88 9_ _ rArhc(^\rSrSrSS/r0rSr\rS\4U4Sjjr Sr Sr S r S r S rS rS \R"S\\\\4S\4Sjr\S(S\R0S \R"S\\R4S\\R0S\\\R"S\\R"S\\\\\4S\\S\\R0S\\S\\S\\S\\S\\R0S\\\R44S\R4S\\S\\\44$S jj5r S)S!jr!\"S\R4S"\S#\S$\RFS\R4S%\4 S&j5r$S'r%U=r&$)*Llama4ForConditionalGenerationirBrkr)cj>[TU]U5 [UR5Ul[ U5Ul[UR5Ul URRUl URRbURROSUl UR5 g)NrE)r-r.rhr.rr,multi_modal_projectorrr`rr}r)r|rr<s r?r.'Llama4ForConditionalGeneration.__init__s  -f.B.BC%>v%F"/0B0BC ,,778< 8P8P8\DKK44bd rAc6URR5$rf)rrrs r?r3Llama4ForConditionalGeneration.get_input_embeddingss""7799rAc:URRU5 grf)rrrs r?r3Llama4ForConditionalGeneration.set_input_embeddingss 007rAc6URR5$rf)rrrs r?r4Llama4ForConditionalGeneration.get_output_embeddingss""88::rAc:URRU5 grf)rrrs r?r4Llama4ForConditionalGeneration.set_output_embeddingss 11.ArAc:URRU5 grf)rr rs r?r *Llama4ForConditionalGeneration.set_decoders ''0rAc6URR5$rf)rr rs r?r *Llama4ForConditionalGeneration.get_decoders""..00rArvision_feature_layervision_feature_select_strategyc US;a[SUR35eUR5VVs0sH upVUcM XV_M nnnUR"U4SS0UD6nURnU$s snnf)a Obtains image last hidden states from the vision tower and apply al projection. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. vision_feature_layer (`Union[int, List[int]]`): The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). )rrz$Unexpected select feature strategy: rF)rritemsrr) r=rrrrkrz image_outputsrns r?get_image_features1Llama4ForConditionalGeneration.get_image_featuress{. *1D DCDDgDgChij j#)<<>C>41Q$!$>C)),]U]V\] $66 Ds A+A+rrrr\rrrQr,rrr'r image_sizesrrCc U bU OURRn U bU OURRn U bU OURRn UbUOURRR nUbUOURRR nUSLUSL- (a [S5eUbUb [S5eUcUR5"U5nUGb_URUUUUS9nURnURSURS55nURU5nXRR:HRS5nUR!UR"5nURSURS55nUSR%S5nUR'5nUURS5:wa [SUS URS535eURS5R)SURS55nUR+UU5nURU5nUR,"SUUUUU U U U UUS . UD6nUSnSnU Gb>UbUSS2URS S - *S24R!UR"5nUS SS2SS24UR!UR"5S:gR/5nU S S S24UR!U R"5S:gR/5n O1US SS2SS24R/5nU S S S24R/5n [0R2"5n!U!"URSURS55U RS5R!UR"55nU (dU4US S-n"UbU4U"-$U"$[5UUUR6UR8UR:UbWS 9$SS 9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, LlavaForConditionalGeneration >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf") >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf") >>> prompt = "USER: \nWhat's the content of the image? ASSISTANT:" >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, text=prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(**inputs, max_new_tokens=15) >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "USER: \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed" ```NrzdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one)rrrrrE).rrzMismatch: final_mask wants z0 embeddings, but multi_modal_projector returned ) rrr\rrQr,rrr'rr!.)rrr\rBrrrS)r)r,rrr.rrrrrrrHrrimage_token_idrrrrsumrmasked_scatterrrr4CrossEntropyLossrr\rBr)#r=rrrrr\rrrrrQr,rrr'rrrr4original_inputs_embeds_shape vision_flatprojected_vision_flatspecial_image_mask final_mask final_mask_1dnum_tokens_to_fill expanded_maskrWrrshift_attention_mask shift_logits shift_labelsloss_fctrs# r?rO&Llama4ForConditionalGeneration.forward s[b2C1N-TXT_T_TqTq$8$D $++JjJj &1%+> ((--b.2E2Eb2IJK$($>$>{$K !"+{{/I/I"I!T!TUW!X +..}/C/CDJ)..r=3E3Eb3IJM&v.66r:M!.!2!2!4 !%:%?%?%BB 12D1EF::O:T:TUV:W9XZ *33B7>>r=CUCUVXCYZM)88H]^M)../KLM%%  )%+'/!5#))      )(6a6<<?Q;N9O9Q6Q'R'U'UV\VcVc'd$%c3B3k23G3J3J6==3Y]^3^_jjl %c12g/C/F/Fv}}/UYZ/Z[ffh %c3B3k2==? %c12g99; **,H!!"l&7&7&;O>OPR>S>V>VWcWjWj>kDY,F'+'7D7V# CV C+#33!//))2>2J   QU   rAc fURR"U4UUUUUS.UD6n USS:XaXIS'U $)N)r\rrr'rrr)rprepare_inputs_for_generation) r=rr\rrrr'rr model_inputss r?r!*C(K*' E*..I** 0Y\j\q\qK!##jjqA 5<< >S>STWeWmWmnprsWtt tK%dD!Q&67>>z1bRTUK))//1 ,2226 *1aL[L+@ANSTVZ\`bcScDdDgDg&&E   ,q0 5@Aq,;,AV5W5c5c 6 Aq!\k\12rA)rrr|rr})NNNNNNNNNNNNNNrN)NNNNNN)'rQrRrSrTrrrr"rnr.rrrrr r r6rYrrrrrrr@rrUrFrrrrrOrrrrrVrWrXs@r?rrs13MNHL | :8;B11''$CcN3), <'+*.1537=A59@D8<-1$(,0/3&*5934$(#U ##U ''U !. U u//0 U "$u'8'8"9: U   1 12U 'uS$s)^'<=U )1 U ))*U D>U $D>U 'tnU d^U !!1!12U c5<</0!U "\\#U $*+%U & u22 3'U U t<4 444{{ 4  4  44rAr)r[ryrhrr)r.)`r7 dataclassesrtypingrrrrrr6torch.nnr4torch.nn.functionalrr(/transformers.models.llama4.configuration_llama4r activationsr cache_utilsr r r generationrintegrations.hub_kernelsrmodeling_attn_mask_utilsrmodeling_flash_attention_utilsrmodeling_outputsrrrrmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrrrr configuration_llama4r"r#!torch.nn.attention.flex_attentionr$integrations.flex_attentionr% get_loggerrQr5Moduler'rZrkrrrrUrrrr|r r rrBr[ryrrrrr,rBrDrTrZr\rerkrrrrrhr__all__rSrAr?rs  !99 N!BB)C>BmmKF&hh@ !!;J   H % B)BII)$ !uxx != =(_-)"BII)".)"X > 2  2  2|| 2 5<< %& 2 UU\\ U# U%,, U(% II% <<% % << % U\\* %  %%D% II% <<% % << % U\\* %  %%4`)"))`)FFRYYFR$KO$K$KN\+\\~ ?,j>m -m ` $<;$< $)BII>)B bii )ryy)XW "))W tbii(6"))6,C -C Ll%:Ol^  rA