§
    9²¸j¤Ö  ã            &       ó,	  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7 d„ Z8 ed¦  «        d~d„¦   «         Z9dej:        de;dej:        fd„Z<	 dd ej=        d!ej:        d"ej:        d#ej:        d$ej:        dz  d%e>d&e>d'e*e,         fd(„Z? ee9¦  «         G d)„ d*ej=        ¦  «        ¦   «         Z@d+ej:        d,e;fd-„ZAd.„ ZBd/„ ZCd0„ ZD ed1d2¦  «        	 	 d€dej:        d3ej:        d4ejE        d5ejE        dz  d6eFdz  f
d7„¦   «         ZG ed8d2¦  «        	 	 d€dej:        d4ejE        d5ejE        dz  d6eFdz  fd9„¦   «         ZH ed:d;¦  «        dd e>d<¦  «        fd=d>dd?ddddd@fdAej:        dBej:        dCej:        dz  dDej:        dEej:        dFej:        dGe;dHej:        dz  dIeIe>e>f         dJeJd6eFdKej:        dz  dLe>dMej:        dz  dNej:        dz  dOe;dz  dPe;dQeJf$dR„¦   «         ZK edSd;¦  «        	 	 	 	 d�dTej:        dej:        dUej:        dEej:        dVej:        dWej:        dFej:        dz  dDej:        dz  dXeJdYej:        dz  fdZ„¦   «         ZL ed[d;¦  «        dddd=d e>d<¦  «        fd=fdej:        dUej:        dEej:        dVej:        dWej:        dGe;dFej:        dz  dDej:        dz  dHej:        dz  dXeJdIeIe>e>f         dJeJfd\„¦   «         ZM eeHeGeKeLeMg¦  «         G d]„ d^ej=        ¦  «        ¦   «         ZN G d_„ d`ej        j=        ¦  «        ZO G da„ dbej=        ¦  «        ZP G dc„ ddej=        ¦  «        ZQ G de„ dfej=        ¦  «        ZRe G dg„ dhej=        ¦  «        ¦   «         ZS G di„ djej=        ¦  «        ZT G dk„ dled=¬m¦  «        ZU edn¦  «         G do„ dpej=        ¦  «        ¦   «         ZV G dq„ dre¦  «        ZWe- G ds„ dte(¦  «        ¦   «         ZXe- G du„ dveX¦  «        ¦   «         ZY	 	 	 d‚dxej:        eIej:                 z  dz  dye;dz  d$ej:        dz  dej:        e;z  fdz„ZZe- G d{„ d|eXe¦  «        ¦   «         Z[g d}¢Z\dS )ƒé    )ÚCallable)Ú	TypedDictN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚGraniteMoeHybridConfigc                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      úŽ/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/granitemoehybrid/modeling_granitemoehybrid.pyÚrotate_halfr1   4   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'ó    Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer1   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r0   Úapply_rotary_pos_embr=   ;   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr2   Úhidden_statesÚn_repÚreturnc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)r*   ÚexpandÚreshape)r>   r?   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r0   Ú	repeat_kvrH   U   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr2   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr'   r   r&   )r)   Údtype)ÚpÚtrainingr#   )rH   Únum_key_value_groupsr+   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32ÚtorS   rP   rU   Ú
contiguous)rJ   rK   rL   rM   rN   rO   rP   rQ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r0   Úeager_attention_forwardrb   a   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r2   c                   óÌ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        dej        dz  de	dz  d	e
ej        ej        f         dz  d
ee         de
ej        ej        f         fd„Zˆ xZS )ÚGraniteMoeHybridAttentionuæ   Hybrid variant that handles ``position_embeddings is None`` â€” granitemoe-hybrid configs can
    opt out of RoPE via ``position_embedding_type=None``, in which case the model passes ``None``
    instead of a ``(cos, sin)`` tuple.ÚconfigÚ	layer_idxc                 ó¨  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j
        | _        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )NrG   T©Úbias)ÚsuperÚ__init__re   rf   ÚgetattrÚhidden_sizeÚnum_attention_headsrG   rE   rV   Úattention_multiplierrO   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfre   rf   Ú	__class__s      €r0   rk   z"GraniteMoeHybridAttention.__init__€   s>  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ2ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr2   Nr>   rN   Úpast_key_valuesÚposition_embeddingsrQ   r@   c                 ó&  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|�|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr&   r#   r'   rI   )rP   rO   )r*   rG   rt   ÚviewrX   ru   rv   r=   Úupdaterf   r   Úget_interfacere   Ú_attn_implementationrb   rU   rp   rO   rC   r]   rw   )ry   r>   rN   r{   r|   rQ   Úinput_shapeÚhidden_shapeÚquery_statesr^   r_   r8   r9   Úattention_interfacera   r`   s                   r0   Úforwardz!GraniteMoeHybridAttention.forward—   sÆ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ*Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r2   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   Úintrk   r+   ÚTensorr	   Útupler   r   r†   Ú__classcell__©rz   s   @r0   rd   rd   z   så   ø€ € € € € ð*ð *ð
Ð5ð 
À#ð 
ð 
ð 
ð 
ð 
ð 
ð6 )-ØHLð%)ð %)à”|ð%)ð œ tÑ+ð%)ð  ™ð	%)ð
 # 5¤<°´Ð#=Ô>ÀÑEð%)ð Ð+Ô,ð%)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð%)ð %)ð %)ð %)ð %)ð %)ð %)ð %)r2   rd   Úinput_tensorÚpad_sizec                 ó¦   — t          | j        ¦  «        dk    r
ddddd|ddfnddd|ddf}t          j        j                             | |dd¬¦  «        S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)ÚmoderM   )Úlenr*   r+   r   rY   Úpad)r‘   r’   Ú	pad_shapes      r0   Úpad_tensor_by_sizerš   Â   sj   € õ 47°|Ô7IÑ3JÔ3JÈaÒ3OÐ3O��A�q˜!˜Q ¨!¨QÐ/Ð/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€IåŒ8Ô×"Ò" <°ÀÐSTÐ"ÑUÔUÐUr2   c                 ó"  — t          | |¦  «        } t          | j        ¦  «        dk    r.|                      | j        d         d|| j        d         ¦  «        S |                      | j        d         d|| j        d         | j        d         ¦  «        S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r&   r'   )rš   r—   r*   rC   )r‘   r’   Ú
chunk_sizes      r0   Úreshape_into_chunksr�   Í   s’   € õ & l°HÑ=Ô=€Lå
ˆ<ÔÑÔ !Ò#Ð#à×#Ò# LÔ$6°qÔ$9¸2¸zÈ<ÔK]Ð^_ÔK`ÑaÔaÐað ×#Ò#ØÔ˜qÔ! 2 z°<Ô3EÀaÔ3HÈ,ÔJ\Ð]^ÔJ_ñ
ô 
ð 	
r2   c                 ó  — |                       d¦  «        } | d         j        g |                       ¦   «         ¢|‘R Ž } t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                      | d¦  «        } t          j        | d¬¦  «        }t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                     | t          j	         ¦  «        }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r&   ©.N©ÚdevicerS   )Údiagonalr   éþÿÿÿr(   )
ÚsizerB   r+   ÚtrilÚonesr¡   ÚboolÚmasked_fillÚcumsumÚinf)r‘   rœ   ÚmaskÚtensor_segsums       r0   Úsegment_sumr­   á   só   € ð ×"Ò" 2Ñ&Ô&€Jð 2�< 	Ô*Ô1ÐS°<×3DÒ3DÑ3FÔ3FÐSÈ
ÐSÐSÐS€LåŒ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqsÐtÑtÔt€DØ×+Ò+¨T¨E°1Ñ5Ô5€Lå”L °2Ð6Ñ6Ô6€Mõ Œ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqrÐsÑsÔs€DØ!×-Ò-¨t¨eµe´i°ZÑ@Ô@€MØÐr2   c                 ób   — |�,| j         }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    N)rS   r\   )r>   rN   rS   s      r0   Úapply_mask_to_padding_statesr¯   õ   sC   € ð
 Ð!ØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐr2   Úcausal_conv1d_updateÚcausal_conv1dÚ
conv_stateÚweightri   Ú
activationc                 óº  — | j         \  }}}|j         d         }t          j        || gd¬¦  «                             |j        ¦  «        }	|                     |	d d …d d …| d …f         ¦  «         t          j        |	|                     d¦  «        |d|¬¦  «        }
|
d d …d d …| d …f         }
|�t          |         |
¦  «        }
|
                     | j        ¦  «        S )Nr&   r(   r#   r   )ÚpaddingÚgroups)
r*   r+   r,   r\   rS   Úcopy_ÚFÚconv1dr5   r   )r>   r²   r³   ri   r´   Ú_rm   Úseq_lenÚ	state_lenÚhidden_states_newÚouts              r0   r°   r°     sî   € ð ,Ô1Ñ€A€{�GØÔ  Ô$€Iåœ	 :¨}Ð"=À2ÐFÑFÔF×IÒIÈ&Ì,ÑWÔWÐØ×ÒÐ& q q q¨!¨!¨!¨i¨Z¨[¨[Ð'8Ô9Ñ:Ô:Ð:Ý
Œ(Ð$ f×&6Ò&6°qÑ&9Ô&9¸4ÈÐS^Ð
_Ñ
_Ô
_€CØ
ˆaˆaˆa����W�H�I�IˆoÔ
€CØÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r2   Úcausal_conv1d_fnc                 ó@  — | j         \  }}}|j         d         dz
  }t          j        |                      |j        ¦  «        |                     d¦  «        |||¬¦  «        d d …d d …d |…f         }	|�t          |         |	¦  «        }	|	                     | j        ¦  «        S )Nr&   r#   )r³   ri   r¶   r·   )r*   r¹   rº   r\   rS   r5   r   )
r>   r³   ri   r´   rQ   r»   rm   r¼   r¶   r¿   s
             r0   rÀ   rÀ     s²   € ð ,Ô1Ñ€A€{�GØŒl˜2Ô Ñ"€Gå
Œ(Ø×Ò˜œÑ&Ô&Ø×Ò Ñ"Ô"ØØØðñ ô ð €a€aˆˆˆˆHˆWˆH€nô€Cð ÐÝ�ZÔ  Ñ%Ô%ˆØ�6Š6�-Ô%Ñ&Ô&Ð&r2   Ú mamba_split_conv1d_scan_combinedÚ	mamba_ssmrª   FÚsiluç�íµ ÷Æ°>TÚzxbcdtÚconv1d_weightÚconv1d_biasÚdt_biasÚAÚDrœ   Úinitial_statesÚdt_limitÚreturn_final_statesÚrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gatec                 ó   — d S ©N© )rÆ   rÇ   rÈ   rÉ   rÊ   rË   rœ   rÌ   rÍ   rÎ   r´   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rQ   s                      r0   Ú!mamba2_split_conv1d_scan_combinedrÙ   ,  s	   € ð, ˆ4r2   Úselective_state_updateÚstateÚdtÚBÚCÚdt_softplusÚzc
                 óH  — |j         \  }}}|j         d         }|j         d         }|�||                     |j        ¦  «        z   }|rt          j        |¦  «        }|d         }t          j        |                     ¦   «         |                     ¦   «         z  ¦  «                             | j        ¬¦  «        }| 	                    ||d|¦  «        }| 
                    ||||z  |¦  «                             ¦   «         }| 	                    ||d|¦  «        }||z  }||d         z                       | j        ¬¦  «        }| |z  |z   }|                      |                     | j        ¦  «        ¦  «         | 	                    ||d|¦  «        }| 
                    ||||z  |¦  «                             ¦   «         }| 	                    |||¦  «        }|                     |j        |j        ¬¦  «        }|                     ||z  ||¦  «        }|                     ||z  |d¦  «        }t          j        ||¦  «        }|                     |||¦  «        }|� |||z  z                        |j        ¦  «        }|	�|t          j        |	¦  «        z  }|                     |j        ¦  «        S )Nr#   r&   rŸ   ©r¡   r    )r*   r\   rS   r¹   Úsoftplusr+   ÚexpÚfloatr¡   rC   rB   r]   r¸   r~   ÚbmmrÄ   )rÛ   r>   rÜ   rÊ   rÝ   rÞ   rË   rÉ   rß   rà   rQ   Ú
batch_sizeÚ	num_headsrG   Ú
num_groupsÚ
state_sizeÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedr¿   s                          r0   Úmamba2_selective_state_updaterñ   E  sq  € ð '4Ô&9Ñ#€J�	˜8Ø”˜”€JØ”˜”€JàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆØ	ˆIŒ€Bõ 
Œ�2—8’8‘:”: §¢¡	¤	Ñ)Ñ	*Ô	*×	-Ò	-°U´\Ð	-Ñ	BÔ	B€Bð 	
�	Š	�*˜j¨!¨ZÑ8Ô8€AØ	�Š�˜Z¨°jÑ)@À*ÑMÔM×XÒXÑZÔZ€AØ	�	Š	�*˜i¨¨JÑ7Ô7€AØ	ˆa‰€Bð �˜iÔ(Ñ(×
,Ò
,°E´LÐ
,Ñ
AÔ
A€Cð ˜‘˜cÑ!€JØ	‡K‚K�
—’˜eœkÑ*Ô*Ñ+Ô+Ð+ð 	
�	Š	�*˜j¨!¨ZÑ8Ô8€AØ	�Š�˜Z¨°jÑ)@À*ÑMÔM×XÒXÑZÔZ€AØ	�	Š	�*˜i¨Ñ4Ô4€Að —’ a¤h°a´g�Ñ>Ô>€JØ$Ÿ/š/¨*°yÑ*@À(ÈJÑWÔWÐØ—’˜
 YÑ.°
¸AÑ>Ô>€JÝ
Œ)Ð'¨Ñ
4Ô
4€CØ
�(Š(�:˜y¨(Ñ
3Ô
3€Cð 	€}Ø�] QÑ&Ñ&×*Ò*¨3¬9Ñ5Ô5ˆà€}Ø•A”F˜1‘I”I‰oˆà�6Š6�-Ô%Ñ&Ô&Ð&r2   Úmamba_chunk_scan_combinedc                 ó	  ‡‡#— | j         \  }}}}|j         d         }|�||                     |j        ¦  «        z   }|	rt          j        |¦  «        }t          j        ||
d         |
d         ¬¦  «        }|                      ¦   «         } |                     ¦   «                              ||z  d|¬¦  «        }|                     ¦   «                              ||z  d|¬¦  «        }‰|‰z  z
  ‰z  Š#d }|�|d         t          | ‰#¦  «        z  }| |d                              ¦   «         z  } |                     | j        ¦  «        |                     ¦   «         z  }ˆˆ#fd„| |||fD ¦   «         \  } }}}| 
                    dddd¦  «        }t          j        |d	¬
¦  «        }t          j        t          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z                       d	¬
¦  «        }|d         | 
                    ddddd¦  «        d         z                       d	¬
¦  «        }|d         | d d …d d …d f         z                       d¬
¦  «        }t          j        |d d …d d …d d …d	d …f         |z
  ¦  «        }|| 
                    ddd	d¦  «        d         z  }|dd d d …f         | d         z                       d¬
¦  «        }|�+|d d …d f                              |j        |j        ¬¦  «        nt          j        |d d …d d…f         ¦  «        }t          j        ||gd¬
¦  «        }t          j        t          t          j        |d d …d d …d d …d	f         d¦  «        ¦  «        ¦  «                             dd¦  «        }|d         |d d …d d …d df         z                       d¬
¦  «        }|d d …d d	…f         |d d …d	f         }}t          j        |¦  «        }|dd d d …f         |d d …d d …d df         z  } |                      d	¦  «        | 
                    dddd¦  «        d         z  }!||!z   }"|"                     |d	||¦  «        }"|�|"|z   }"‰#dk    r|"d d …d |…f         }"|r|"|fS |"S )Nr'   r   r#   )ÚminÚmax)r)   Úoutput_sizerŸ   c                 ó2   •— g | ]}t          |‰‰¦  «        ‘ŒS rØ   )r�   )Ú.0Útensorrœ   r’   s     €€r0   ú
<listcomp>z%mamba2_chunk_scan.<locals>.<listcomp>ª  s'   ø€ ÐwÐwÐwÐTZÕ1°&¸(ÀJÑOÔOÐwÐwÐwr2   r   r&   r(   r”   r£   .©rS   r¡   )r#   r   ).NN)r*   r\   rS   r¹   rã   r+   Úclamprå   Úrepeat_interleaverš   Úpermuter©   rä   r­   Úsumr¡   Ú
zeros_liker,   r˜   rX   rC   )$r>   rÜ   rÊ   rÝ   rÞ   rœ   rË   rÉ   rÌ   rß   rÍ   rÎ   rQ   rç   Úsequence_lengthrè   rG   ré   Ú
D_residualÚA_cumsumÚLÚGÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesÚfinal_stateÚstate_decay_outÚC_times_statesÚY_offÚoutputr’   s$        `                             @r0   Úmamba2_chunk_scanr  ƒ  s  øø€ ð  8EÔ7JÑ4€J� ¨HØ”˜”€JàÐØ�'—*’*˜RœXÑ&Ô&Ñ&ˆØð ÝŒZ˜‰^Œ^ˆÝ	Œ�R˜X aœ[¨h°q¬kÐ	:Ñ	:Ô	:€Bà!×'Ò'Ñ)Ô)€MØ	�Š‰	Œ	×#Ò# I°Ñ$;ÀÐPYÐ#ÑZÔZ€AØ	�Š‰	Œ	×#Ò# I°Ñ$;ÀÐPYÐ#ÑZÔZ€Aà˜_¨zÑ9Ñ9¸ZÑG€HØ€JØ€}Ø�y”\Õ$6°}ÀhÑ$OÔ$OÑOˆ
ð " B y¤M×$7Ò$7Ñ$9Ô$9Ñ9€MØ	�Šˆ]Ô Ñ!Ô! B§H¢H¡J¤JÑ.€Að xÐwÐwÐwÐwÐ_lÐnoÐqrÐtuÐ^vÐwÑwÔwÑ€M�1�a˜à	�	Š	�!�Q˜˜1ÑÔ€AÝŒ|˜A 2Ð&Ñ&Ô&€Hõ 	Œ	•+˜a‘.”.Ñ!Ô!€Að 
ˆ1ˆ1ˆ1ˆaˆaˆa����D˜!˜!˜!˜Q˜Q˜QÐÔ	 ! A A A q q q¨$°°°°1°1°1°a°a°aÐ$7Ô"8Ñ	8×=Ò=À"Ð=ÑEÔE€Að 
ˆ9Œ˜Ÿ	š	 ! Q¨¨1¨aÑ0Ô0°Ô;Ñ	;×@Ò@ÀRÐ@ÑHÔH€Að �	Œl˜]¨1¨1¨1¨a¨a¨a°¨:Ô6Ñ6×;Ò;ÀÐ;ÑBÔB€Fõ ”9˜X a a a¨¨¨¨A¨A¨A¨r¨s¨s lÔ3°hÑ>Ñ?Ô?€LØ�,×&Ò& q¨"¨b°!Ñ4Ô4°YÔ?Ñ?€GØ�c˜4   �lÔ# m°IÔ&>Ñ>×CÒCÈÐCÑJÔJ€Fð Ð%ð 	�q�q�q˜$�wÔ×"Ò"¨¬¸f¼mÐ"ÑLÔLÐLåÔ˜f Q Q Q¨¨¨ UœmÑ,Ô,ð õ
 ŒY˜¨Ð0°aÐ8Ñ8Ô8€FÝ”)�K­¬¨h°q°q°q¸!¸!¸!¸Q¸Q¸QÀ°{Ô.CÀVÑ(LÔ(LÑMÔMÑNÔN×XÒXÐYZÐ\]Ñ^Ô^€KØ˜oÔ.°¸¸¸¸1¸1¸1¸dÀC¸Ô1HÑH×MÒMÐRSÐMÑTÔT€JØ$ Q Q Q¨¨¨ VÔ,¨j¸¸¸¸B¸Ô.?ˆK€Fõ ”i Ñ)Ô)€OØ�s˜D ! ! !�|”_ v¨a¨a¨a°°°°D¸#¨oÔ'>Ñ>€NØ×Ò˜rÑ"Ô" _×%<Ò%<¸QÀÀ1ÀaÑ%HÔ%HÈÔ%SÑS€Eð �e‰^€FØ�^Š^˜J¨¨I°xÑ@Ô@€FàÐØ˜*Ñ$ˆð �!‚|€|Ø˜˜˜Ð+˜OÐ+Ð+Ô,ˆàð #Ø�{Ð"Ð"à€Mr2   c            	       ó¾   ‡ — e Zd ZdZddededefˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
 ed¦  «        	 	 dd
ej        ded	z  dej        d	z  fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridMambaLayeruk  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - Some extra variables that our layer doesn't need have been removed
    Tre   rf   Úinitialize_mixer_weightsc           	      óN  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        t          |j        | j        z  ¦  «        | _        || _        |j        | _        |j        | _        t&          |j                 | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j        d| j        z  | j        z  z   | _        t?          j         | j        | j        |j        | j
        | j        | j
        dz
  ¬¦  «        | _!        | j        | j        z   | j        z   }t?          j"        | j        || j        ¬¦  «        | _#        t?          j$        tK          j&        | j        ¦  «        ¦  «        | _'        t?          j$        tK          j&        | j        ¦  «        ¦  «        | _(        tS          | j        | j        ¬¦  «        | _*        t?          j$        tK          j&        | j        ¦  «        ¦  «        | _+        |r)| j'        j,        j-        dk    r|  .                    ¦   «          t?          j"        | j        | j        | j        ¬¦  «        | _/        |j0        |         | _1        d S )Nr'   r#   )Úin_channelsÚout_channelsri   Úkernel_sizer·   r¶   rh   ©ÚepsÚmeta)2rj   rk   Úmamba_proj_biasÚuse_biasÚmamba_n_headsrè   rm   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizerŒ   Úmamba_expandÚintermediate_sizerf   Úmamba_conv_biasÚuse_conv_biasÚ
hidden_actr´   r   ÚactÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headrG   Úmamba_chunk_sizerœ   Útime_step_limitÚconv_dimr   ÚConv1drº   rr   Úin_projÚ	Parameterr+   ÚemptyrÉ   ÚA_logÚGraniteMoeHybridRMSNormGatedÚnormrË   r¡   ÚtypeÚinit_granitemoehybrid_weightsÚout_projÚlayer_typesÚ
layer_type)ry   re   rf   r  Úprojection_sizerz   s        €r0   rk   z#GraniteMoeHybridMambaLayer.__init__ú  s8  ø€ Ý‰Œ×ÒÑÔÐØÔ.ˆŒØÔ-ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ"ˆŒØ#Ô3ˆÔØ Ô+ˆŒÝ˜&Ô+Ô,ˆŒØ"(Ô"5ˆÔØÔ-ˆŒØÔ+ˆŒØ Ô1ˆŒà%Ô5ˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒÝ”iØœØœØÔ'ØÔ-Ø”=ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØ”ð
ñ 
ô 
ˆŒõ ”|¥E¤K°´Ñ$?Ô$?Ñ@Ô@ˆŒõ ”\¥%¤+¨d¬nÑ"=Ô"=Ñ>Ô>ˆŒ
Ý0°Ô1GÈTÔMdÐeÑeÔeˆŒ	Ý”�eœk¨$¬.Ñ9Ô9Ñ:Ô:ˆŒØ#ð 	1¨¬Ô(;Ô(@ÀFÒ(JÐ(JØ×.Ò.Ñ0Ô0Ð0Ýœ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒà Ô,¨YÔ7ˆŒˆˆr2   c                 ó*  — t          j        d| j        dz   | j        j        t           j        ¬¦  «        }t          j        | j        t          j        |¦  «        ¦  «         t          j	        | j
        ¦  «         t          j	        | j        ¦  «         d S )Nr#   r    )r+   Úarangerè   r7  r¡   r[   Úinitr¸   ÚlogÚones_rË   rÉ   )ry   rÊ   s     r0   r;  z8GraniteMoeHybridMambaLayer.init_granitemoehybrid_weights,  sp   € åŒL˜˜DœN¨QÑ.°t´zÔ7HÕPUÔP]Ð^Ñ^Ô^ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐÝŒ
�4”<Ñ Ô Ð Ð Ð r2   rº   Nr>   Úcache_paramsrN   c                 ór
  — |j         \  }}}|j        }|d uo|                     | j        ¦  «        }	t	          ||¦  «        }|                      |¦  «        }
t          j        | j         	                    ¦   «         ¦  «         }|d| j
        iz  }| j        r�|€Žt          |
| j        j                             d¦  «        | j        j        | j        |f| j        | j        | j        | j        j        | j        j        | j        j        | j        j        | j        | j        dddœ|¤Ž}|�|S |
                     | j        | j        | j        gd¬¦  «        \  }}}|	r:|j        | j                 j        d         }|j        | j                 j         d         }| !                    dd¦  «        }|	r^|dk    rX|j        | j                 j"        sAtG          ||| j        j                             d¦  «        | j        j        | j        ¬	¦  «        }nu|�"| $                    || j        | j%        ¬
¦  «        }tM          || j        j                             d¦  «        | j        j        fd| j        i|¤Ž}|�|d d …d d …| d …f         }t	          | !                    dd¦  «        |¦  «        }t          j        || j        | j        | j'        z  | j        | j'        z  gd¬¦  «        \  }}}|	�rI|dk    �rB| (                    || j        | j        ¦  «        }| !                    dd¦  «         )                    dd| j        ¦  «        }|d d …d d f          )                    d| j        | j'        ¦  «        }| (                    || j        | j'        ¦  «        }| (                    || j        | j'        ¦  «        }| j        d d …d f          )                    d| j        ¦  «        }| j        d d …d f          )                    d| j        ¦  «        }tU          |||||||fd |ddœ|¤Ž}| (                    |dd¦  «        }nÓ|d u}tW          | (                    ||| j        | j        ¦  «        ||| (                    ||| j        | j'        ¦  «        | (                    ||| j        | j'        ¦  «        f| j        | j        d || j        d|	r|nd | j
        dœ|¤Ž}|r"|\  }}| ,                    || j        ¬¦  «         n|}| -                    ||d¦  «        }|                      ||¦  «        }|                      | .                    |¦  «        ¦  «        }|S )NrÍ   r#   F)rË   rœ   r´   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rÎ   r&   r(   r   r'   )r´   )r$  r´   T)rà   rÉ   rß   )rœ   rË   rà   rÎ   rÉ   rß   rÌ   rÍ   )rf   )/r*   rS   Úhas_previous_staterf   r¯   r4  r+   rä   r7  rå   r1  rU   rÙ   rº   r³   Úsqueezeri   rÉ   rË   rœ   r´   r9  Úvariance_epsilonr<  rG   r.  Úsplitr&  r2  rè   ÚlayersÚconv_statesÚrecurrent_statesrX   Úrecord_pastr°   Úupdate_conv_stater$  rÀ   r"  r~   rB   rñ   r  Úupdate_recurrent_staterC   r\   )ry   r>   rE  rN   rQ   rç   r¼   r»   rS   Úuse_precomputed_statesÚprojected_statesrÊ   Úfused_kwargsÚfused_outputÚgateÚhidden_states_B_CrÜ   r²   Úrecurrent_staterÝ   rÞ   rË   rÉ   Úscan_outputÚoutput_final_stateÚscan_resultr  Úcontextualized_statess                               r0   r†   z"GraniteMoeHybridMambaLayer.forward3  sÍ  € ð "/Ô!4Ñˆ
�G˜QØÔ#ˆØ!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6ÐåŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ ¨TÔ-AÐ BÑBˆØŒ=ð 	$˜\Ð1Ý<Ø Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”Øðð ”&Øœ?Øœ?Ø#œyÔ/Ø œIÔ6Ø#œ}Ô3Ø!œ]Ô/ØœØœØ!&Ø$)ð!ð ð" ð#ð ˆLð* Ð'Ø#Ð#à&6×&<Ò&<ØÔ# T¤]°D´NÐCÈð '=ñ '
ô '
Ñ#ˆÐ ð "ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð .×7Ò7¸¸1Ñ=Ô=ÐØ!ð 	G g°¢l l¸<Ô;NÈtÌ~Ô;^Ô;j lÝ 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Øœ?ð!ñ !ô !ÐÐð Ð'Ø$0×$BÒ$BØ%Ø”NØ%)Ô%:ð %Cñ %ô %Ð!õ !1Ø!Ø”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô ð!ð !ð  œ?ð	!ð
 ð!ð !Ðð Ð'Ø$5°a°a°a¸¸¸¸W¸H¸I¸I°oÔ$FÐ!õ 9Ð9J×9TÒ9TÐUVÐXYÑ9ZÔ9ZÐ\jÑkÔkÐÝ#œkØØÔ# T¤]°TÔ5HÑ%HÈ$Ì-ÐZ^ÔZmÑJmÐnØð
ñ 
ô 
Ñˆ�q˜!ð "ñ 2	G g°¢l¡lØ)×.Ò.¨z¸4¼>È4Ì=ÑYÔYˆMØ—’˜a Ñ#Ô#×*Ò*¨2¨r°4´=ÑAÔAˆBØ�!�!�!�T˜4�-Ô ×'Ò'¨¨D¬M¸4Ô;NÑOÔOˆAØ—’�z 4¤=°$Ô2EÑFÔFˆAØ—’�z 4¤=°$Ô2EÑFÔFˆAØ”�q�q�q˜$�w”×&Ò& r¨4¬=Ñ9Ô9ˆAØ”l 1 1 1 d 7Ô+×2Ò2°2°t´}ÑEÔEˆGå7ØØØØØØØðð ØØ ðð ð ðð ˆKð &×*Ò*¨:°q¸"Ñ=Ô=ˆKˆKð ".°TÐ!9ÐÝ+Ø×"Ò" :¨w¸¼ÈÌÑVÔVØØØ—’�z 7¨D¬M¸4Ô;NÑOÔOØ—’�z 7¨D¬M¸4Ô;NÑOÔOðð  œ?Ø”&ØØ$6ØœØ Ø2HÐR˜˜ÈdØÔ-ðð ð ðð ˆKð" "ð *Ø+6Ñ(�˜[Ø×3Ò3°KÈ4Ì>Ð3ÑZÔZÐZÐZà)�à%×-Ò-¨j¸'À2ÑFÔFˆKà—i’i ¨TÑ2Ô2ˆð !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r2   )Tr‡   )rˆ   r‰   rŠ   r‹   r$   rŒ   r§   rk   r+   Úno_gradr;  r   r�   r	   r†   r�   r�   s   @r0   r  r  å  sõ   ø€ € € € € ð	ð 	ð08ð 08Ð5ð 08À#ð 08Ðaeð 08ð 08ð 08ð 08ð 08ð 08ðd €U„]�_„_ð!ð !ñ „_ð!ð Ð˜HÑ%Ô%ð &*Ø.2ð	P%ð P%à”|ðP%ð ˜d‘lðP%ð œ tÑ+ð	P%ð P%ð P%ñ &Ô%ðP%ð P%ð P%ð P%ð P%r2   r  c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )r8  rÅ   c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        d S r×   ©rj   rk   r   r5  r+   r¦   r³   rI  ©ry   rm   r  rz   s      €r0   rk   z%GraniteMoeHybridRMSNormGated.__init__È  sB   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr2   Nc                 óœ  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|                     d¦  «                             dd¬¦  «        }|t          j	        || j
        z   ¦  «        z  }| j        |                     |¦  «        z  S ©Nr'   r&   T)Úkeepdim)rS   r\   r+   r[   r   rY   rÄ   ÚpowÚmeanÚrsqrtrI  r³   )ry   r>   rU  Úinput_dtypeÚvariances        r0   r†   z$GraniteMoeHybridRMSNormGated.forwardÍ  sª   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆàÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆàŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r2   ©rÅ   r×   )rˆ   r‰   rŠ   rk   r†   r�   r�   s   @r0   r8  r8  Ç  sQ   ø€ € € € € ð$ð $ð $ð $ð $ð $ð
	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;r2   r8  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGraniteMoeHybridMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    re   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        t          j	        | j        | j        dz  d¬¦  «        | _
        t          j	        | j        | j        d¬¦  «        | _        d S )Nr'   Frh   )rj   rk   rm   Ú
input_sizeÚshared_intermediate_sizer   r)  r´   r   rr   Úinput_linearÚoutput_linear©ry   re   rz   s     €r0   rk   zGraniteMoeHybridMLP.__init__â  s…   ø€ Ý‰Œ×ÒÑÔÐà Ô,ˆŒØ!Ô:ˆÔÝ  Ô!2Ô3ˆŒÝœI d¤o°tÔ7GÈ!Ñ7KÐRWÐXÑXÔXˆÔÝœY tÔ'7¸¼ÈuÐUÑUÔUˆÔÐÐr2   r>   r@   c                 óÐ   — |                       |¦  «        }|                     dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr'   r&   r(   r   r#   )ro  Úchunkr´   rp  )ry   r>   Úchunked_hidden_statess      r0   r†   zGraniteMoeHybridMLP.forwardë  sj   € Ø×)Ò)¨-Ñ8Ô8ˆØ -× 3Ò 3°A¸2Ð 3Ñ >Ô >ÐØŸšÐ(=¸aÔ(@ÑAÔAÐDYÐZ[ÔD\Ñ\ˆØ×*Ò*¨=Ñ9Ô9ˆØÐr2   ©
rˆ   r‰   rŠ   r‹   r$   rk   r+   r�   r†   r�   r�   s   @r0   rk  rk  Ù  s|   ø€ € € € € ðð ðVÐ5ð Vð Vð Vð Vð Vð Vð U¤\ð °e´lð ð ð ð ð ð ð ð r2   rk  c            	       óð   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        	 d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridRotaryEmbeddingr¡   z5.18)ÚversionNre   c                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)rj   rk   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenre   Úrope_parametersrz  Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)ry   re   r¡   Úrope_init_fnr„  rz   s        €r0   rk   z(GraniteMoeHybridRotaryEmbedding.__init__ô  sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐr2   r@   c                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarG   Ng      ð?r   r'   ©rS   )r€  rl   rm   rn   r+   rA  rå   r\   )re   r¡   rQ   Úbaser)   Úattention_factorr„  s          r0   r�  z?GraniteMoeHybridRotaryEmbedding.compute_default_rope_parameters  sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r2   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r&   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr'   r(   rŠ  )r„  rå   rB   r*   r\   r¡   Ú
isinstancer:  Ústrr    rX   r+   r,   r8   r‚  r9   rS   )
ry   r-   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr�  ÚfreqsÚembr8   r9   s
             r0   r†   z'GraniteMoeHybridRotaryEmbedding.forward  s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*r×   )rˆ   r‰   rŠ   r   r$   rk   ÚstaticmethodrŽ   r+   r�   rå   r�  r\  r   r†   r�   r�   s   @r0   rw  rw  ó  sø   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð OÐ5ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.à/3ð5ð 5Ø&ð5à	ˆuŒ|˜UÐ"Ô	#ð5ð 5ð 5ñ /Ô.ñ „\ð5ð( €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r2   rw  c                   ór   ‡ — e Zd ZdZdefˆ fd„Zdej        deej        ej        ej        f         fd„Z	ˆ xZ
S )ÚGraniteMoeHybridTopKRoutera¡  Top-k gating that returns the routing decisions without grouping tokens by expert.

    Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live
    here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the
    experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm``
    paths can compile cleanly.
    re   c                 óä   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        t          j	        | j        |j
        ¦  «        ¦  «        | _        d S r×   )rj   rk   Únum_local_expertsÚnum_expertsÚnum_experts_per_tokÚtop_kr   r5  r+   r6  rm   r³   rq  s     €r0   rk   z#GraniteMoeHybridTopKRouter.__init__4  sU   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØÔ/ˆŒ
Ý”l¥5¤;¨tÔ/?ÀÔASÑ#TÔ#TÑUÔUˆŒˆˆr2   r>   r@   c                 óô   — t          j        || j        ¦  «                             ¦   «         }|                     | j        d¬¦  «        \  }}t          j        |d¬¦  «                             |¦  «        }|||fS )Nr&   r(   )	r¹   Úlinearr³   rå   Útopkr   r+   rZ   Útype_as)ry   r>   Úrouter_logitsÚtop_k_logitsÚtop_k_indexÚtop_k_weightss         r0   r†   z"GraniteMoeHybridTopKRouter.forward:  so   € Ýœ °´Ñ<Ô<×BÒBÑDÔDˆØ$1×$6Ò$6°t´zÀrÐ$6Ñ$JÔ$JÑ!ˆ�kÝœ l¸Ð;Ñ;Ô;×CÒCÀMÑRÔRˆØ˜M¨=Ð8Ð8r2   )rˆ   r‰   rŠ   r‹   r$   rk   r+   r�   rŽ   r†   r�   r�   s   @r0   r›  r›  +  sŽ   ø€ € € € € ðð ðVÐ5ð Vð Vð Vð Vð Vð Vð9 U¤\ð 9°e¸E¼LÈ%Ì,ÐX]ÔXdÐ<dÔ6eð 9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r2   r›  c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGraniteMoeHybridExpertsz2Collection of expert weights stored as 3D tensors.re   c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr'   )rj   rk   r�  rž  rm   Ú
hidden_dimr&  Úintermediate_dimr   r5  r+   r6  Úgate_up_projÚ	down_projr   r)  Úact_fnrq  s     €r0   rk   z GraniteMoeHybridExperts.__init__E  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô 8ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr2   r>   r§  r¨  r@   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr'   r#   r   )r&   r£   r(   r&   )r+   r   r\  r   rY   Úone_hotrž  rþ   Úgreaterrÿ   ÚnonzeroÚwherer¢  r®  rs  r°  r¯  Ú
index_add_r\   rS   )ry   r>   r§  r¨  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_staterU  ÚupÚcurrent_hidden_statess                 r0   r†   zGraniteMoeHybridExperts.forwardN  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6ru  r�   s   @r0   rª  rª  A  s�   ø€ € € € € à<Ð<ð0Ð5ð 0ð 0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r2   rª  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGraniteMoeHybridMoEzISparsely-gated mixture-of-experts block: router decides, experts compute.re   c                 ó°   •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          |¦  «        | _        d S r×   )rj   rk   rm   rm  r›  Úrouterrª  Úexpertsrq  s     €r0   rk   zGraniteMoeHybridMoE.__init__l  sE   ø€ Ý‰Œ×ÒÑÔÐØ Ô,ˆŒÝ0°Ñ8Ô8ˆŒÝ.¨vÑ6Ô6ˆŒˆˆr2   Úlayer_inputr@   c                 óö   — |                      ¦   «         \  }}}|                     d|¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }	|	                     ||| j        ¦  «        S )Nr&   )r¤   rC   rÄ  rÅ  r~   rm  )
ry   rÆ  ÚbszÚlengthÚemb_sizer>   r§  r¨  r»   Úlayer_outputs
             r0   r†   zGraniteMoeHybridMoE.forwardr  sv   € Ø +× 0Ò 0Ñ 2Ô 2ÑˆˆV�XØ#×+Ò+¨B°Ñ9Ô9ˆØ(,¯ª°MÑ(BÔ(BÑ%ˆ�] AØ—|’| M°;ÀÑNÔNˆØ× Ò   f¨d¬oÑ>Ô>Ð>r2   ru  r�   s   @r0   rÂ  rÂ  i  sq   ø€ € € € € ØSÐSð7Ð5ð 7ð 7ð 7ð 7ð 7ð 7ð? 5¤<ð ?°E´Lð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?r2   rÂ  c                   ód   — e Zd ZU dZej        ed<   ej        ed<   eed<   eed<   ej        ed<   dS )ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_kÚseq_idxN)	rˆ   r‰   rŠ   r‹   r+   Ú
LongTensorÚ__annotations__rŒ   Ú	IntTensorrØ   r2   r0   rÍ  rÍ  z  sb   € € € € € € ðð ð  Ô#Ð#Ð#Ñ#ØÔ#Ð#Ð#Ñ#ØÐÐÑØÐÐÑØŒ_ÐÐÑÐÐr2   rÍ  )ÚtotalÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚGraniteMoeHybridRMSNormrÅ   r  r@   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zF
        GraniteMoeHybridRMSNorm is equivalent to T5LayerNorm
        Nr_  r`  s      €r0   rk   z GraniteMoeHybridRMSNorm.__init__”  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr2   r>   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S rb  )	rS   r\   r+   r[   rd  re  rf  rI  r³   )ry   r>   rg  rh  s       r0   r†   zGraniteMoeHybridRMSNorm.forwardœ  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r2   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rŽ   r³   r*   rI  )ry   s    r0   Ú
extra_reprz"GraniteMoeHybridRMSNorm.extra_repr£  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr2   ri  )
rˆ   r‰   rŠ   rå   rk   r+   r�   r†   rÝ  r�   r�   s   @r0   rÙ  rÙ  ’  sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr2   rÙ  c                   ó  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 ddej        dej        dz  de	dz  d	e
dz  d
eej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridDecoderLayerre   rf   c                 óN  •— t          ¦   «                              ¦   «          |j        | _        d | _        t	          |j        |j        ¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        |j        dk    rt          |¦  «        nd | _
        |j        | _        t          |¦  «        | _        d | _        |j        |         dk    rt!          ||¦  «        | _        nt#          ||¦  «        | _        |j        |         | _        t'          |dd¦  «        dk    | _        d S )Nr  r   Úlinear_attentionr�  )rj   rk   rm   Ú	self_attnrÙ  r+  Úinput_layernormÚpost_attention_layernormr�  rÂ  Úblock_sparse_moeÚresidual_multiplierrk  Ú
shared_mlpÚmambaÚlayers_block_typer  rd   Ú
block_typerl   Úhas_expertsrx   s      €r0   rk   z%GraniteMoeHybridDecoderLayer.__init__¨  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔàˆŒÝ6°vÔ7IÈvÔObÐcÑcÔcˆÔÝ(?ÀÔ@RÐX^ÔXkÐ(lÑ(lÔ(lˆÔ%ð @FÔ?WÐZ[Ò?[Ð?[Õ 3°FÑ ;Ô ;Ð ;ÐaeˆÔØ#)Ô#=ˆÔ Ý-¨fÑ5Ô5ˆŒØˆŒ
àÔ# IÔ.Ð2DÒDÐDÝ3°F¸IÑFÔFˆDŒJˆJå6°v¸yÑIÔIˆDŒNØ Ô2°9Ô=ˆŒõ # 6Ð+>ÀÑBÔBÀQÒFˆÔÐÐr2   NFr>   rN   r{   Ú	use_cacher|   rQ   r@   c           	      óˆ  — |}|                       |¦  «        }| j        � | j        d|||dœ|¤Ž}n | j        d|||||dœ|¤Ž\  }}||| j        z  z   }|}|                      |¦  «        }| j        r.|                      |¦  «        }	|	|                      |¦  «        z   }n|                      |¦  «        }||| j        z  z   }|S )N)r>   rE  rN   )r>   rN   r{   rì  r|   rØ   )rã  rè  râ  ræ  rä  rë  rå  rç  )
ry   r>   rN   r{   rì  r|   rQ   Úresidualr»   Úmoe_hidden_statess
             r0   r†   z$GraniteMoeHybridDecoderLayer.forward¿  s  € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆàŒ:Ð!Ø&˜DœJð Ø+Ø,Ø-ðð ð ð	ð ˆMˆMð  .˜tœ~ð  Ø+Ø-Ø /Ø#Ø$7ð ð  ð ð ð  ÑˆM˜1ð ! =°4Ô3KÑ#KÑKˆØ ˆØ×5Ò5°mÑDÔDˆàÔð 	;Ø $× 5Ò 5°mÑ DÔ DÐØ-°·²ÀÑ0NÔ0NÑNˆMˆMà ŸOšO¨MÑ:Ô:ˆMà  =°4Ô3KÑ#KÑKˆØÐr2   ©NNFN)rˆ   r‰   rŠ   r$   rŒ   rk   r   r+   r�   r	   r§   rŽ   r   rÍ  ÚFloatTensorr†   r�   r�   s   @r0   rß  rß  §  s  ø€ € € € € ðGÐ5ð GÀ#ð Gð Gð Gð Gð Gð Gð. ð /3Ø(,Ø!&ØHLð(ð (à”|ð(ð œ tÑ+ð(ð  ™ð	(ð
 ˜$‘;ð(ð # 5¤<°´Ð#=Ô>ÀÑEð(ð Ð4Ô5ð(ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð(ð (ð (ñ „^ð(ð (ð (ð (ð (r2   rß  c                   óŠ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridPreTrainedModelre   ÚmodelTrß  r{   )r>   Ú
attentionsc           
      ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rMt	          j        |j        d| j        j        ¬¦  «         t	          j        |j	        d| j        j        ¬¦  «         n;t          |t          ¦  «        r&t	          j        |j        d| j        j        ¬¦  «         t          |t          ¦  «        r{t	          j        |j        ¦  «         t	          j        |j        t#          j        t#          j        d|j        dz   ¦  «        ¦  «        ¦  «         t	          j        |j        ¦  «         d S t          |t,          ¦  «        rt	          j        |j        ¦  «         d S d S )NrI   )re  Ústdr#   )rj   Ú_init_weightsr’  rª  rB  Únormal_r®  re   Úinitializer_ranger¯  r›  r³   r  rD  rÉ   r¸   r7  r+   rC  rA  rè   rË   r8  )ry   rJ   rz   s     €r0   rø  z-GraniteMoeHybridPreTrainedModel._init_weightsý  sG  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ5Ñ6Ô6ð 	UÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÝ˜Õ :Ñ;Ô;ð 	UÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝ�fÕ8Ñ9Ô9ð 	&ÝŒJ�v”~Ñ&Ô&Ð&ÝŒJ�v”|¥U¤Y­u¬|¸A¸vÔ?OÐRSÑ?SÑ/TÔ/TÑ%UÔ%UÑVÔVÐVÝŒJ�v”xÑ Ô Ð Ð Ð Ý˜Õ <Ñ=Ô=ð 	&ÝŒJ�v”}Ñ%Ô%Ð%Ð%Ð%ð	&ð 	&r2   )rˆ   r‰   rŠ   r$   rÔ  Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrß  rd   Ú_can_record_outputsÚ_is_statefulr+   r\  rø  r�   r�   s   @r0   ró  ró  ë  s­   ø€ € € € € € à"Ð"Ð"Ñ"ØÐØ&*Ð#Ø7Ð8ÐØ#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&Ðà5Ø/ðð Ðð €Là€U„]�_„_ð&ð &ð &ð &ñ „_ð&ð &ð &ð &ð &r2   ró  c                   óè   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridModelre   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        ‰j        dk    rt#          ‰¦  «        nd | _        d| _        ‰j        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rØ   )rß  )rø   rf   re   s     €r0   rú   z2GraniteMoeHybridModel.__init__.<locals>.<listcomp>  s$   ø€ ÐnÐnÐnÀÕ)¨&°)Ñ<Ô<ÐnÐnÐnr2   r  ÚropeF)rj   rk   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrm   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrK  rÙ  r+  r9  Úposition_embedding_typerw  Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initrq  s    `€r0   rk   zGraniteMoeHybridModel.__init__  sò   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØnÐnÐnÐnÍeÐTZÔTlÑNmÔNmÐnÑnÔnñ
ô 
ˆŒõ ,¨FÔ,>ÀFÔDWÐXÑXÔXˆŒ	ØEKÔEcÐgmÒEmÐEmÕ9¸&ÑAÔAÐAÐswˆŒØ&+ˆÔ#Ø$*Ô$?ˆÔ!ð 	�ŠÑÔÐÐÐr2   NÚ	input_idsrN   r”  r{   Úinputs_embedsrì  rQ   r@   c           	      óØ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|| j        z  }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }t          |x}	t          ¦  «        s%| j        |||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}d }| j        �|                      ||¦  «        }t!          | j        ¦  «        D ])\  }} ||f|	| j        j        |                  |||dœ|¤Ž}Œ*|                      |¦  «        }t)          ||¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)re   r   r#   râ   )re   r  rN   r{   )Úfull_attentionrá  )rN   r{   rì  r|   )Úlast_hidden_stater{   rØ   )Ú
ValueErrorr  r  r
   re   Úget_seq_lengthr+   rA  r*   r¡   r5   r’  Údictr   r   r  Ú	enumeraterK  ré  r9  r   )ry   r  rN   r”  r{   r  rì  rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr>   r|   ÚiÚdecoder_layers                  r0   r†   zGraniteMoeHybridModel.forward   sí  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà%¨Ô(AÑAˆàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2ð	ð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &ˆØ"ÐØŒ?Ð&Ø"&§/¢/°-ÀÑ"NÔ"NÐå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3PÐQRÔ3SÔTØ /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r2   )NNNNNN)rˆ   r‰   rŠ   r$   rk   r   r!   r"   r+   rÓ  r�   r	   rñ  r§   r   rÍ  rŽ   r   r†   r�   r�   s   @r0   r  r    s  ø€ € € € € ðÐ5ð ð ð ð ð ð ð" ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð4Ô5ð<
ð 
Ð(Ñ	(ð<
ð <
ð <
ñ „_ñ  Ôñ „^ð<
ð <
ð <
ð <
ð <
r2   r  r'   Úgate_logitsrž  c           	      óÎ  — | �t          | t          ¦  «        sdS | d         j        }t          j        |t          j        |¬¦  «        }t          j        |t          j        |¬¦  «        }d}|�4|                     d¦  «                             |t          j        ¬¦  «        }| D �]‰}	t          j        j	         
                    |	                     |¦  «        d¬¦  «        }
t          j        |
|d¬¦  «        \  }}|€z|t          j        |                     d¦  «        |¬¦  «                             ¦   «         z   }||
                     ¦   «                              d¬¦  «        z   }||
j        d         z   }ŒÒ|t          j        |t          j        |¬¦  «                             d|                     d¦  «        |                     |¦  «        ¦  «        z   }||
                     ¦   «         |                     d¦  «        z                       d¬¦  «        z   }||                     ¦   «         z   }�Œ‹||z  }||z  }t          j        ||                     d¦  «        z  ¦  «        }||z  S )	aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rû   rI   r&   r    r(   )Ú	minlength)r’  rŽ   r¡   r+   Úzerosr[   rC   r\   r   rY   rZ   r£  Úbincountrå   rÿ   r*   Úscatter_add_rý   r5   )r&  rž  r   rN   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsr»   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   r0   Úload_balancing_loss_funcr7  b  sV  € ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqð ! ”^Ô*€NÝ!œK¨½5¼=ÐQ_Ð`Ñ`Ô`ÐÝ”k +µU´]È>ÐZÑZÔZ€OØ€JàÐ!à"×*Ò*¨2Ñ.Ô.×1Ò1¸ÍuÌ}Ð1Ñ]Ô]ˆ	à!ð 6ñ 6ˆ
Ýœ(Ô-×5Ò5°j·m²mÀNÑ6SÔ6SÐY[Ð5Ñ\Ô\ˆÝ#œj¨¸%ÀRÐHÑHÔHÑˆÐØÐ!ð &­¬Ð7G×7OÒ7OÐPRÑ7SÔ7SÐ_jÐ(kÑ(kÔ(k×(qÒ(qÑ(sÔ(sÑsð "ð .°×0EÒ0EÑ0GÔ0G×0KÒ0KÐPQÐ0KÑ0RÔ0RÑRˆOØ# oÔ&;¸AÔ&>Ñ>ˆJˆJð %:½E¼KØ¥5¤=¸ð=ñ =ô =çŠl˜1Ð.×6Ò6°rÑ:Ô:¸I×<WÒ<WÐX]Ñ<^Ô<^Ñ_Ô_ñ%`Ð!ð .°×1FÒ1FÑ1HÔ1HÈ9×K^ÒK^Ð_aÑKbÔKbÑ1b×0gÒ0gÐlmÐ0gÑ0nÔ0nÑnˆOØ# i§m¢m¡o¤oÑ5ˆJ‰Jà-°
Ñ:ÐØ,¨zÑ9Ðå”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r2   c                   ó  ‡ — e Zd ZddiZddiZddgdgfiZddiZdefˆ fd	„Ze	e
	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej        z  deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr>   ÚlogitsÚkeep_full_weightre   c                 ó^  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |j	        | _	        |j
        | _        |j        | _        |j        | _        |                      ¦   «          d S )NFrh   )rj   rk   r  rô  r  r   rr   rm   r:  Úrouter_aux_loss_coefr�  rž  rŸ  Úlogits_scalingr  rq  s     €r0   rk   z$GraniteMoeHybridForCausalLM.__init__®  s–   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý*¨6Ñ2Ô2ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ$*Ô$?ˆÔ!Ø!Ô3ˆÔØ#)Ô#=ˆÔ Ø$Ô3ˆÔð 	�ŠÑÔÐÐÐr2   Nr   r  rN   r”  r{   r  ÚlabelsÚoutput_router_logitsÚlogits_to_keepr@   c	           	      ó2  — |�|n| j         j        } | j        d|||||dœ|	¤Ž}
|
j        }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }|| j         j        z  }d}|� | j	        ||fd| j         j
        i|	¤Ž}d}|rHt          |
j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t%          ||||
j        |
j        |
j        |
j        ¬¦  «        S )a–  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm-granite/granite-4.0-h-tiny")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-4.0-h-tiny")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r  rN   r”  r{   r  r  )ÚlossÚaux_lossr<  r{   r>   rõ  r¥  rØ   )re   rB  rô  r  r’  rŒ   Úslicer:  r@  Úloss_functionr  r7  r¥  rž  rŸ  r?  r\   r¡   r   r{   r>   rõ  )ry   r  rN   r”  r{   r  rA  rB  rC  rQ   Úoutputsr>   Úslice_indicesr<  rE  rF  s                   r0   r†   z#GraniteMoeHybridForCausalLM.forward»  s‘  € ðJ %9Ð$DÐ Ð È$Ì+ÔJjð 	ð �$”*ð 
ØØ)Ø%Ø+Ø'ð
ð 
ð ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ˜$œ+Ô4Ñ4ˆàˆØÐà%�4Ô%ØØðð ð  œ;Ô1ðð ð	ð ˆDð ˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�Ý(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r2   )NNNNNNNr   )rˆ   r‰   rŠ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr$   rk   r   r   r+   rÓ  r�   r	   rñ  r§   rŒ   rŽ   r   r†   r�   r�   s   @r0   r9  r9  §  sk  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€JðÐ5ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø-.ðQ
ð Q
àÔ# dÑ*ðQ
ð œ tÑ+ðQ
ð Ô&¨Ñ-ð	Q
ð
  ™ðQ
ð Ô(¨4Ñ/ðQ
ð Ô  4Ñ'ðQ
ð # T™kðQ
ð ˜eœlÑ*ðQ
ð 
Ð*Ñ	*ðQ
ð Q
ð Q
ñ Ôñ „^ðQ
ð Q
ð Q
ð Q
ð Q
r2   r9  )r9  r  ró  )r#   )rI   r‡   rð  )Nr'   N)]Úcollections.abcr   Útypingr   r+   Útorch.nn.functionalr   rY   r¹   Ú r   rB  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_granitemoehybridr$   r1   r=   r�   rŒ   rH   ÚModulerå   rb   rd   rš   r�   r­   r¯   r5  r“  r°   rÀ   rŽ   r§   rÙ   rñ   r  r  r8  rk  rw  r›  rª  rÂ  rÍ  rÙ  rß  ró  r  r7  r9  Ú__all__rØ   r2   r0   ú<module>re     sþ
  ðð* %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð >Ð =Ð =Ð =Ð =Ð =Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø BÐ BÐ BÐ BÐ BÐ Bð(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ðA)ð A)ð A)ð A)ð A) ¤	ñ A)ô A)ñ +Ô*ðA)ðNV U¤\ð V¸Sð Vð Vð Vð Vð
ð 
ð 
ð(ð ð ð(	ð 	ð 	ð (Ð'Ð(>ÀÑPÔPð
 !%Ø!ð'ð 'Ø”<ð'à”ð'ð ŒLð'ð Œ,˜Ñ
ð	'ð
 �d‘
ð'ð 'ð 'ñ QÔPð'ð& (Ð'Ð(:¸OÑLÔLð !%Ø!ð	'ð 'Ø”<ð'àŒLð'ð Œ,˜Ñ
ð'ð �d‘
ð	'ð 'ð 'ñ MÔLð'ð, (Ð'Ð(JÈKÑXÔXð +/Ø%(¨%¨%°©,¬,Ð$7Ø %ØØ*.ØØ*.Ø(,ØØØ!ð%ð ØŒLðà”<ðð ” Ñ$ðð Œ\ð	ð
 „|ðð „|ðð ðð ”L 4Ñ'ðð �E˜5�LÔ!ðð ðð ðð ”L 4Ñ'ðð ðð ”L 4Ñ'ðð ”, Ñ%ðð  �4‰Zð!ð" ð#ð$ ð%ð ð ñ YÔXðð0 (Ð'Ð(@À+ÑNÔNð "Ø#'ØØ!ð:'ð :'ØŒ<ð:'à”<ð:'ð 	Œð:'ð „|ð	:'ð
 „|ð:'ð „|ð:'ð „|�dÑð:'ð Œ\˜DÑ ð:'ð ð:'ð „|�dÑð:'ð :'ð :'ñ OÔNð:'ðz (Ð'Ð(CÀ[ÑQÔQð "Ø#'Ø*.ØØ%(¨%¨%°©,¬,Ð$7Ø %ð^ð ^Ø”<ð^àŒð^ð „|ð^ð „|ð	^ð
 „|ð^ð ð^ð „|�dÑð^ð Œ\˜DÑ ð^ð ”L 4Ñ'ð^ð ð^ð �E˜5�LÔ!ð^ð ð^ð ^ð ^ñ RÔQð^ðB ÐàØØ)Ø%Øðñô ðV%ð V%ð V%ð V%ð V% ¤ñ V%ô V%ñô ðV%ðr;ð ;ð ;ð ;ð ; 5¤8¤?ñ ;ô ;ð ;ð$ð ð ð ð ˜"œ)ñ ô ð ð45<ð 5<ð 5<ð 5<ð 5< b¤iñ 5<ô 5<ð 5<ðp9ð 9ð 9ð 9ð 9 ¤ñ 9ô 9ð 9ð, ð$#ð $#ð $#ð $#ð $#˜bœiñ $#ô $#ñ Ôð$#ðN?ð ?ð ?ð ?ð ?˜"œ)ñ ?ô ?ð ?ð"ð ð ð ð  )°5ð ñ ô ð ð0 Ð˜YÑ'Ô'ðJð Jð Jð Jð J˜bœiñ Jô Jñ (Ô'ðJð(Að Að Að Að AÐ#=ñ Aô Að AðH ð&ð &ð &ð &ð & oñ &ô &ñ „ð&ðB ðQ
ð Q
ð Q
ð Q
ð Q
Ð;ñ Q
ô Q
ñ „ðQ
ðl #Ø
Ø*.ð	B&ð B&Ø”  e¤lÔ 3Ñ3°dÑ:ðB&à�t‘ðB&ð ”L 4Ñ'ð	B&ð
 „\�CÑðB&ð B&ð B&ð B&ðJ ðf
ð f
ð f
ð f
ð f
Ð"AÀ?ñ f
ô f
ñ „ðf
ðR fÐ
eÐ
e€€€r2   