§
    9²¸j%¼  ã                   ó¦  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  e(j1        e2¦  «        Z3	 	 	 d;dej4        dej5        dej5        dej5        dej5        dz  de6dz  de6dej5        dz  de#e%         fd „Z7 G d!„ d"ej4        ¦  «        Z8 G d#„ d$ej4        ¦  «        Z9 G d%„ d&ej4        ¦  «        Z: G d'„ d(ej4        ¦  «        Z; G d)„ d*ej4        ¦  «        Z< G d+„ d,ej4        ¦  «        Z= G d-„ d.ej4        ¦  «        Z> G d/„ d0e¦  «        Z?e& G d1„ d2e!¦  «        ¦   «         Z@ G d3„ d4e@¦  «        ZA G d5„ d6ej4        ¦  «        ZB e&d7¬8¦  «         G d9„ d:e@e¦  «        ¦   «         ZCd:d2gZDdS )<zPyTorch Pop2Piano model.é    N)ÚCallable)Únn)ÚCrossEntropyLoss)ÚGenerationConfigé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚis_torchdynamo_compilingÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚPop2PianoConfigç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚposition_biasÚkwargsc                 ó¼  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j                             |	|| j        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «         	                    ¦   «         }
|
|	fS )Néÿÿÿÿç      à¿é   r   ©Údim)ÚpÚtrainingr   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr(   r2   Ú
contiguous)r"   r#   r$   r%   r&   r'   r(   r)   r*   Úattn_weightsÚattn_outputs              ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/pop2piano/modeling_pop2piano.pyÚeager_attention_forwardr=   -   sØ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ Ø# mÑ3ˆàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚPop2PianoLayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zj
        Construct a layernorm module in the Pop2Piano style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr4   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €r<   rD   zPop2PianoLayerNorm.__init__O   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr>   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Nr.   r,   T)Úkeepdim)Útor4   Úfloat32ÚpowÚmeanÚrsqrtrH   rG   ÚdtypeÚfloat16Úbfloat16)rI   Úhidden_statesÚvariances      r<   ÚforwardzPop2PianoLayerNorm.forwardW   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r>   )rA   )Ú__name__Ú
__module__Ú__qualname__rD   rY   Ú__classcell__©rL   s   @r<   r@   r@   N   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r>   r@   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)rC   rD   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater(   r	   Údense_act_fnÚact©rI   ra   rL   s     €r<   rD   zPop2PianoDenseActDense.__init__i   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr>   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)ri   rn   r(   Ú
isinstancerj   rG   r4   ÚTensorrT   Úint8rO   )rI   rW   s     r<   rY   zPop2PianoDenseActDense.forwardp   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr>   ©rZ   r[   r\   r    rD   rY   r]   r^   s   @r<   r`   r`   h   sS   ø€ € € € € ð/˜ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r>   r`   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseGatedActDensera   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rc   )rC   rD   r   rf   rg   rh   Úwi_0Úwi_1rj   rk   rl   r(   r	   rm   rn   ro   s     €r<   rD   z$Pop2PianoDenseGatedActDense.__init__€   s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr>   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S rq   )rn   ry   rz   r(   rr   rj   rG   r4   rs   rT   rt   rO   )rI   rW   Úhidden_geluÚhidden_linears       r<   rY   z#Pop2PianoDenseGatedActDense.forwardˆ   sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐr>   ru   r^   s   @r<   rw   rw      sS   ø€ € € € € ð/˜ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r>   rw   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoLayerFFra   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©rK   )rC   rD   Úis_gated_actrw   ÚDenseReluDenser`   r@   rg   Úlayer_norm_epsilonÚ
layer_normr   rk   rl   r(   ro   s     €r<   rD   zPop2PianoLayerFF.__init__ž   sy   ø€ Ý‰Œ×ÒÑÔÐØÔð 	AÝ"=¸fÑ"EÔ"EˆDÔÐå"8¸Ñ"@Ô"@ˆDÔå,¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rq   )r…   rƒ   r(   )rI   rW   Úforwarded_statess      r<   rY   zPop2PianoLayerFF.forward¨   sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr>   ru   r^   s   @r<   r   r   �   sS   ø€ € € € € ð7˜ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r>   r   c                   ó|   ‡ — e Zd Z	 	 	 ddededz  defˆ fd„Zedd
„¦   «         Zdd„Z		 	 	 	 dde
e         fd„Zˆ xZS )ÚPop2PianoAttentionFNra   Ú	layer_idxÚ	is_causalc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        || _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        |j        | _        | j        | j
        z  | _        d| _        || _        |€/| j        r(t$                               d| j        j        › d�¦  «         t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        | j        r$t-          j        | j        | j        ¦  «        | _        d| _        d S )Nç      ð?zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Frd   )rC   rD   ra   Ú
is_decoderr‹   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerg   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrl   r(   Ú	inner_dimr'   rŠ   ÚloggerÚwarning_oncerL   rZ   r   rf   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)rI   ra   r�   rŠ   r‹   rL   s        €r<   rD   zPop2PianoAttention.__init__±   s~  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒØ"ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒàˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r>   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r.   r   )rO   r4   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r<   Ú_relative_position_bucketz,Pop2PianoAttention._relative_position_bucketØ   s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr>   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasN)rT   Údevice)r­   r®   r¯   )r.   r   r   r   )rž   rG   r¶   r4   Úaranger£   r´   rŽ   r�   r‘   ÚpermuteÚ	unsqueeze)
rI   Úquery_lengthÚ
key_lengthr¶   Úpast_seen_tokensÚcontext_positionÚmemory_positionr¬   Úrelative_position_bucketÚvaluess
             r<   Úcompute_biaszPop2PianoAttention.compute_bias  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr>   r*   c                 ó"  — |j         dd…         }g |¢d‘| j        ‘R }|�|                     | j        ¦  «        nd}	t	          |	t
          j        ¦  «        r|	                     ¦   «         n|	}	|du}
|                      |¦  «         	                    |¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|
r|j        }n
|j        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t          ¦  «        rd|j        | j        <   |€~|j         d         }| j        sFt          j        d| j        |d         |f|j        |j        ¬	¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |	¬
¦  «        }t?          j         | j!        j"        tF          ¦  «        } || ||||f| j        sdn| j$        | j%        |dœ|¤Ž\  }} |j&        g |¢d‘R Ž  '                    ¦   «         }|  (                    |¦  «        }|||fS )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr,   r   r   r.   FTéþÿÿÿ)r¶   rT   )r¶   r¼   r!   )r(   r'   r)   ))Úshaper“   Úget_seq_lengthrŠ   rr   r4   rs   Úcloner™   Úviewr6   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrÀ   rš   r›   Úupdater�   Úzerosr•   r¶   rT   rŸ   r2   Úrequires_gradrÁ   r   Úget_interfacera   Ú_attn_implementationr=   r(   r'   Úreshaper9   rœ   )rI   rW   ÚmaskÚkey_value_statesr)   Úpast_key_valuesr*   Úinput_shapeÚhidden_shaper¼   Úis_cross_attentionÚquery_statesrÈ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaper»   Úattention_interfacer;   r:   s                         r<   rY   zPop2PianoAttention.forward  s�  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>ð Ð Ø#Ô)¨"Ô-ˆJØÔ3ð Ý %¤Ø˜œ k°!¤n°jÐAØ'Ô.Ø&Ô,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°|Ô7JÐ]mð !2ñ !ô !�õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ'ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—f’f˜[Ñ)Ô)ˆà˜M¨<Ð7Ð7r>   )FNF)Tr    r¡   )Nr   )NNNN)rZ   r[   r\   r    ÚintÚboolrD   Ústaticmethodr´   rÁ   r   r   rY   r]   r^   s   @r<   r‰   r‰   °   sä   ø€ € € € € ð %*Ø $Øð%,ð %,àð%,ð ˜‘:ð	%,ð
 ð%,ð %,ð %,ð %,ð %,ð %,ðN ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØðX8ð X8ð Ð+Ô,ðX8ð X8ð X8ð X8ð X8ð X8ð X8ð X8r>   r‰   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )	ÚPop2PianoLayerSelfAttentionFNrŠ   c                 óþ   •— t          ¦   «                              ¦   «          t          ||||j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
        |j        ¦  «        | _        d S )N©r�   rŠ   r‹   r�   )rC   rD   r‰   rŽ   ÚSelfAttentionr@   rg   r„   r…   r   rk   rl   r(   ©rI   ra   r�   rŠ   rL   s       €r<   rD   z$Pop2PianoLayerSelfAttention.__init__v  sr   ø€ Ý‰Œ×ÒÑÔÐÝ/ØØ(CØØÔ'ð	
ñ 
ô 
ˆÔõ -¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r*   c                 ó’   — |                       |¦  «        } | j        |f|||dœ|¤Ž\  }}}||                      |¦  «        z   }|||fS )N)rÔ   r)   rÖ   )r…   rè   r(   )	rI   rW   r&   r)   rÖ   r*   Únormed_hidden_statesÚattention_outputr:   s	            r<   rY   z#Pop2PianoLayerSelfAttention.forward�  sy   € ð  $Ÿš¨}Ñ=Ô=ÐØ8J¸Ô8JØ ð9
àØ'Ø+ð	9
ð 9
ð
 ð9
ð 9
Ñ5Ð˜-¨ð )¨4¯<ª<Ð8HÑ+IÔ+IÑIÐØ °Ð<Ð<r>   ©FN©NNN©	rZ   r[   r\   rá   rD   r   r   rY   r]   r^   s   @r<   rå   rå   u  s‚   ø€ € € € € ð	7ð 	7ÈSÐSWÉZð 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð ØØð=ð =ð Ð+Ô,ð=ð =ð =ð =ð =ð =ð =ð =r>   rå   c                   óL   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 ddee         fd„Zˆ xZS )ÚPop2PianoLayerCrossAttentionNrŠ   c                 óô   •— t          ¦   «                              ¦   «          t          |d|d¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrç   r�   )rC   rD   r‰   ÚEncDecAttentionr@   rg   r„   r…   r   rk   rl   r(   )rI   ra   rŠ   rL   s      €r<   rD   z%Pop2PianoLayerCrossAttention.__init__—  sm   ø€ Ý‰Œ×ÒÑÔÐÝ1Ø°ÀÐV[ð 
ñ  
ô  
ˆÔõ -¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   r*   c                 ó”   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}}	||                      |¦  «        z   }
|
||	fS )N)rÔ   rÕ   r)   rÖ   )r…   ró   r(   )rI   rW   rÕ   r&   r)   rÖ   r*   rë   rì   r:   Úlayer_outputs              r<   rY   z$Pop2PianoLayerCrossAttention.forwardŸ  s{   € ð  $Ÿš¨}Ñ=Ô=ÐØ8L¸Ô8LØ ð9
àØ-Ø'Ø+ð9
ð 9
ð ð9
ð 9
Ñ5Ð˜-¨ð % t§|¢|Ð4DÑ'EÔ'EÑEˆØ˜]¨LÐ8Ð8r>   rq   rî   rï   r^   s   @r<   rñ   rñ   –  s�   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØð9ð 9ð Ð+Ô,ð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r>   rñ   c                   óR   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 ddee         fd„Zˆ xZS )	ÚPop2PianoBlockFNrŠ   c                 ó’  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          |||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )N©r�   rŠ   )rŠ   )
rC   rD   rŽ   r   Ú
ModuleListÚlayerÚappendrå   rñ   r   ré   s       €r<   rD   zPop2PianoBlock.__init__·  s»   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÝ'ØÐ4OÐ[dðñ ô ñ	
ô 	
ð 	
ð
 Œ?ð 	YØŒJ×ÒÕ:¸6ÈYÐWÑWÔWÑXÔXÐXàŒ
×ÒÕ*¨6Ñ2Ô2Ñ3Ô3Ð3Ð3Ð3r>   r*   c                 ó”  —  | j         d         |f|||dœ|¤Ž\  }}	}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }d }| j
        o|d u}|r¿ | j         d         |f||||dœ|¤Ž\  }}}
|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        } | j         d         |¦  «        }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }||	|fS )Nr   )r&   r)   rÖ   iè  )r¥   Úmaxr   )rÕ   r&   r)   rÖ   r,   )rû   rT   r4   rU   r«   ÚisinfÚanyÚfinforþ   ÚclamprŽ   )rI   rW   r&   r)   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrÖ   r*   Úself_attn_position_biasÚ_Úclamp_valueÚcross_attn_position_biasÚdo_cross_attentions                 r<   rY   zPop2PianoBlock.forwardÅ  sR  € ð 5B°D´J¸q´MØð5
à)Ø'Ø+ð	5
ð 5
ð
 ð5
ð 5
Ñ1ˆÐ.°ð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà#'Ð Ø!œ_ÐRÐ1FÈdÐ1RÐØð 	^Ø9F¸¼ÀA¼Øð:à!6Ø5Ø;Ø /ð:ð :ð ð:ð :Ñ6ˆMÐ3°Qð Ô"¥e¤mÒ3Ð3Ý#œkÝ”K Ñ.Ô.×2Ò2Ñ4Ô4Ý”K Ô 3Ñ4Ô4Ô8¸4Ñ?Ý”K Ô 3Ñ4Ô4Ô8ñô �õ
 !&¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMàÐ5Ð7OÐOÐOr>   rí   )NNNNNNrï   r^   s   @r<   r÷   r÷   ¶  s•   ø€ € € € € ð4ð 4ÈSÐSWÉZð 4ð 4ð 4ð 4ð 4ð 4ð" ØØ"Ø#Ø&*Øð=Pð =Pð Ð+Ô,ð=Pð =Pð =Pð =Pð =Pð =Pð =Pð =Pr>   r÷   c                   óÐ   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dgZ
dZdZdZ eed¬	¦  «         eed
¬	¦  «         eed
¬	¦  «        dœZ ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚPop2PianoPreTrainedModelra   Útransformer)ÚaudioTFr÷   rj   r   )Úindexr,   )rW   Ú
attentionsÚcross_attentionsc                 ó
	  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         dS t	          |t          ¦  «        r&t          j
        |j        j        d|dz  ¬¦  «         dS t	          |t          ¦  «        r\t          j
        |j        j        d|dz  ¬¦  «         t          |d¦  «        r&t          j
        |j        j        d|dz  ¬¦  «         dS dS t	          |t           ¦  «        ræt          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t.          ¦  «        �rVt          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t4          ¦  «        rö| j        j        }| j        j        }| j        j        }t          j
        |j        j        d|||z  dz  z  ¬¦  «         t          j
        |j        j        d||dz  z  ¬¦  «         t          j
        |j        j        d||dz  z  ¬¦  «         t          j
        |j         j        d|||z  dz  z  ¬¦  «         |j!        r+t          j
        |j"        j        d||dz  z  ¬¦  «         dS dS dS )zInitialize the weightsr�   r!   )rR   ÚstdÚlm_headr-   re   N)#rC   Ú_init_weightsra   Úinitializer_factorrr   r@   ÚinitÚ	constant_rG   ÚPop2PianoConcatEmbeddingToMelÚnormal_Ú	embeddingÚ!Pop2PianoForConditionalGenerationÚsharedÚhasattrr  r`   ri   rg   re   Úzeros_rj   rh   rw   ry   rz   r‰   r’   r”   r™   rš   r›   rœ   r�   rž   )rI   r"   Úfactorrg   r“   r•   rL   s         €r<   r  z&Pop2PianoPreTrainedModel._init_weights  s]  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�fÕ0Ñ1Ô1ð "	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7Ý˜Õ =Ñ>Ô>ð  	pÝŒL˜Ô)Ô0°sÀÈÁÐMÑMÔMÐMÐMÐMÝ˜Õ AÑBÔBð 	pÝŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÐOÐOðPð På˜Õ 6Ñ7Ô7ð 	pÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ ;Ñ<Ô<ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 2Ñ3Ô3ð 		pØ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð		pð 		pðpð pr>   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )Nzoself.model.config.decoder_start_token_id has to be defined. In Pop2Piano it is usually set to the pad_token_id..r,   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)ra   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrÄ   rÆ   Úmasked_fill_)rI   Ú	input_idsr#  r$  Úshifted_input_idss        r<   Ú_shift_rightz%Pop2PianoPreTrainedModel._shift_rightD  s»   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð Bñô ð ð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r>   )rZ   r[   r\   r    Ú__annotations__Úbase_model_prefixÚoutput_modalitiesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpar   r÷   rå   rñ   Ú_can_record_outputsr4   Úno_gradr  r*  r]   r^   s   @r<   r  r    sò   ø€ € € € € € àÐÐÑØ%ÐØ"ÐØ&*Ð#à"ÐØ)Ð*ÐØ!˜FÐð !ÐØÐØ€Nð (˜¨¸aÐ@Ñ@Ô@Ø$�nÐ%@ÈÐKÑKÔKØ*˜NÐ+GÈrÐRÑRÔRðð Ðð €U„]�_„_ð&pð &pð &pð &pñ „_ð&pðP!ð !ð !ð !ð !ð !ð !r>   r  c                   ó|   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 	 	 	 	 ddee	         fd„¦   «         ¦   «         ¦   «         Z
ˆ xZS )ÚPop2PianoStackc                 óÌ  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        |                      ¦   «          d| _        d S )Nc           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   rù   )r÷   râ   )Ú.0Úira   s     €r<   ú
<listcomp>z+Pop2PianoStack.__init__.<locals>.<listcomp>b  sC   ø€ ð ð ð àõ ˜vÅ4ÈÈQÊÁ<Ä<Ð[\Ð]Ñ]Ô]ðð ð r>   r�   F)rC   rD   r   r�   Ú
vocab_sizerg   Úembed_tokensrŽ   rú   ÚrangeÚ
num_layersÚblockr@   r„   Úfinal_layer_normrk   rl   r(   Ú	post_initrŸ   ro   s    `€r<   rD   zPop2PianoStack.__init__[  sÓ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒå”]ðð ð ð å˜vÔ0Ñ1Ô1ðñ ô ñ
ô 
ˆŒ
õ !3°6´>ÀvÔG`Ð aÑ aÔ aˆÔÝ”z &Ô"5Ñ6Ô6ˆŒð 	�ŠÑÔÐØ&+ˆÔ#Ð#Ð#r>   c                 ó   — || _         d S rq   )r?  ©rI   Únew_embeddingss     r<   Úset_input_embeddingsz#Pop2PianoStack.set_input_embeddingso  s   € Ø*ˆÔÐÐr>   Nr*   c           	      ó@  — |�#|�!| j         rdnd}	t          d|	› d|	› d�¦  «        ‚|�1|                     ¦   «         }
|                     d|
d         ¦  «        }n@|�|                     ¦   «         d d…         }
n!| j         rdnd}	t          d|	› d|	› d	�¦  «        ‚| j        r%| j        r|rt                               d
¦  «         d}|€+| j        €t          d¦  «        ‚|                      |¦  «        }|
\  }}|du r| j         st          d| › d�¦  «        ‚| j         r]|rZ|€X| j	        j
        r7t          t          | j	        ¬¦  «        t          | j	        ¬¦  «        ¦  «        }nt          | j	        ¬¦  «        }n	| j         sd }|�|                     ¦   «         nd}|€/t          ¦   «         s!||z   }t          j        |||j        ¬¦  «        }| j	        j         rt%          | j	        |||¬¦  «        }nt'          | j	        ||¬¦  «        }|�t'          | j	        |||¬¦  «        }d }d }|                      |¦  «        }| j        D ]#} |||||||fd|i|¤Ž\  }}}|}| j         r|�|}Œ$|                      |¦  «        }|                      |¦  «        }t/          ||¬¦  «        S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer,   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTz)`use_cache` can only be set to `True` if z is used as a decoder)ra   r   ©r¶   )ra   rL  r&   rÖ   )ra   rL  r&   )ra   rL  r&   r  rÖ   )Úlast_hidden_staterÖ   )rŽ   r%  r3   rÇ   rŸ   r2   r—   r˜   r?  ra   Úis_encoder_decoderr   r   rÅ   r   r4   rF   r¶   r   r   r(   rB  rC  r   )rI   r(  r&   r  r  rL  rÖ   Ú	use_cacher*   Úerr_msg_prefixr×   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthÚmask_seq_lengthÚcausal_maskr)   r  rW   Úlayer_moduleÚself_attention_position_biasÚcross_attention_position_biass                         r<   rY   zPop2PianoStack.forwardr  sœ  € ð Ð  ]Ð%>Ø+/¬?ÐB˜Z˜ZÀˆNÝØw¨>ÐwÐwÈÐwÐwÐwñô ð ð Ð"Ø#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKà+/¬?ÐB˜Z˜ZÀˆNÝÐu¸>ÐuÐuÐXfÐuÐuÐuÑvÔvÐvàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ ØÔ Ð(Ý Ð!_Ñ`Ô`Ð`Ø ×-Ò-¨iÑ8Ô8ˆMà!,Ñˆ
�Jà˜ÐÐØ”?ð jÝ Ð!hÈTÐ!hÐ!hÐ!hÑiÔiÐiàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNàŒ;Ô!ð 	Ý,Ø”{Ø+Ø-Ø /ð	ñ ô ˆKˆKõ 4Ø”{Ø+Ø-ðñ ô ˆKð "Ð-Ý%>Ø”{Ø+Ø5Ø&;ð	&ñ &ô &Ð"ð ˆØ(,Ð%àŸš ]Ñ3Ô3ˆà œJð 	Nð 	NˆLØYeÐYeØØØØ%Ø&Ø-ð	Zð 	Zð !0ð	Zð ð	Zð 	ZÑVˆMÐ7Ð9Vð 9ˆMØŒð NÐ#8Ð#DØ0MÐ-øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
r>   )NNNNNNN)rZ   r[   r\   rD   rH  r   r   r   r   r   rY   r]   r^   s   @r<   r8  r8  Y  s²   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð(+ð +ð +ð  ØØð ØØ"Ø#ØØØðp
ð p
ð Ð+Ô,ðp
ð p
ð p
ñ „^ñ „_ñ  Ôðp
ð p
ð p
ð p
ð p
r>   r8  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r  z'Embedding Matrix for `composer` tokens.c                 ó’   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        d S )N)Únum_embeddingsÚembedding_dim)rC   rD   r   r�   Úcomposer_vocab_sizerg   r  ro   s     €r<   rD   z&Pop2PianoConcatEmbeddingToMel.__init__ë  s:   ø€ Ý‰Œ×ÒÑÔÐÝœ°VÔ5OÐ_eÔ_mÐnÑnÔnˆŒˆˆr>   c                 ó�   — ||z
  }|                       |¦  «                             d¦  «        }t          j        ||gd¬¦  «        }|S )Nr   r/   )r  r¹   r4   Úcat)rI   ÚfeatureÚindex_valueÚembedding_offsetÚindex_shiftedÚcomposer_embeddingrL  s          r<   rY   z%Pop2PianoConcatEmbeddingToMel.forwardï  sM   € Ø#Ð&6Ñ6ˆØ!Ÿ^š^¨MÑ:Ô:×DÒDÀQÑGÔGÐÝœ	Ð#5°wÐ"?ÀQÐGÑGÔGˆØÐr>   )rZ   r[   r\   Ú__doc__rD   rY   r]   r^   s   @r<   r  r  è  sR   ø€ € € € € Ø1Ð1ðoð oð oð oð oðð ð ð ð ð ð r>   r  zA
    Pop2Piano Model with a `language modeling` head on top.
    )Úcustom_introc                   óö  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Z	 ddej	        d	e
d
edej	        dz  fd„Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej        dz  deeej                          dz  dedz  dej	        dz  dej	        dz  dej	        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Z ej        ¦   «         	 	 	 dˆ fd„	¦   «         Zdej        fd„Zˆ xZS )r  zshared.weight)zencoder.embed_tokens.weightzdecoder.embed_tokens.weightra   c                 ó6  •— t          ¦   «                              |¦  «         || _        |j        | _        t          j        |j        |j        ¦  «        | _        t          |¦  «        | _
        t          j        |¦  «        }d|_        d|_        t          |¦  «        | _        t          j        |¦  «        }d|_        |j        |_        t          |¦  «        | _        t          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrd   )rC   rD   ra   rg   Ú	model_dimr   r�   r>  r  r  Úmel_conditionerÚcopyÚdeepcopyrŽ   rP  r8  ÚencoderÚnum_decoder_layersrA  Údecoderrf   r  rD  )rI   ra   Úencoder_configÚdecoder_configrL   s       €r<   rD   z*Pop2PianoForConditionalGeneration.__init__  sç   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒå<¸VÑDÔDˆÔåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ å% nÑ5Ô5ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý% nÑ5Ô5ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr>   c                 ó   — | j         S rq   )r  )rI   s    r<   Úget_input_embeddingsz6Pop2PianoForConditionalGeneration.get_input_embeddings  s
   € ØŒ{Ðr>   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rq   )r  rn  rH  rp  rF  s     r<   rH  z6Pop2PianoForConditionalGeneration.set_input_embeddings  s;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r>   NÚinput_featuresÚcomposerÚgeneration_configr&   c                 ó<  — |j         }||vr4t          dt          |                     ¦   «         ¦  «        › d|› �¦  «        ‚||         }t	          j        || j        ¬¦  «        }|                     |j        d         ¦  «        }t          | 
                    ¦   «         ¦  «        }|                      |||¬¦  «        }|�\d||dd…df                              ¦   «          <   t	          j        |dd…df                              dd	¦  «        |gd	¬
¦  «        }||fS |dfS )aã  
        This method is used to concatenate mel conditioner tokens at the front of the input_features in order to
        control the type of MIDI token generated by the model.

        Args:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                input features extracted from the feature extractor.
            composer (`str`):
                composer token which determines the type of MIDI tokens to be generated.
            generation_config (`~generation.GenerationConfig`):
                The generation is used to get the composer-feature_token pair.
            attention_mask (``, *optional*):
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
        zPlease choose a composer from z. Composer received - rM  r   )ra  rb  rc  Nr!   r,   r   )Úaxis)Úcomposer_to_feature_tokenr%  ÚlistrÍ   r4   Útensorr¶   ÚrepeatrÄ   r¥   rÀ   rk  râ   ÚconcatenaterÇ   )rI   rv  rw  rx  r&   r{  Úcomposer_valuerc  s           r<   Úget_mel_conditioner_outputsz=Pop2PianoForConditionalGeneration.get_mel_conditioner_outputs"  sN  € ð0 %6Ô$OÐ!ØÐ4Ð4Ð4ÝØyµÐ6O×6TÒ6TÑ6VÔ6VÑ1WÔ1WÐyÐyÐowÐyÐyñô ð ð 3°8Ô<ˆÝœ n¸T¼[ÐIÑIÔIˆØ'×.Ò.¨~Ô/CÀAÔ/FÑGÔGˆåÐ8×?Ò?ÑAÔAÑBÔBÐà×-Ò-Ø"Ø&Ø-ð .ñ 
ô 
ˆð
 Ð%Ø;>ˆN˜N¨1¨1¨1¨a¨4Ô0×5Ò5Ñ7Ô7Ð7Ñ8õ #Ô.°¸q¸q¸qÀ!¸tÔ0D×0IÒ0IÈ"ÈaÑ0PÔ0PÐR`Ð/aÐhiÐjÑjÔjˆNØ! >Ð1Ð1à˜tÐ#Ð#r>   r(  Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrÖ   rL  Údecoder_inputs_embedsÚlabelsrP  r*   Úreturnc                 ó(  — |�|�t          d¦  «        ‚|�|€|}|€ | j        d|||dœ|¤Ž}nct          |t          ¦  «        sNt          |d         t	          |¦  «        dk    r|d         ndt	          |¦  «        dk    r|d         nd¬¦  «        }|d         }|
�|€|	€|                      |
¦  «        } | j        d|||	||||dœ|¤Ž}|d         }| j        j        r|| j	        d	z  z  }|  
                    |¦  «        }d}|
�Vt          d
¬¦  «        } ||                     d|                     d¦  «        ¦  «        |
                     d¦  «        ¦  «        }t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aq  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Pop2Piano is a model with relative position embeddings
            so you should be able to pad the inputs on both the right and the left. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail.
            [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining
            take a look a [Pop2Piano Training](./Pop2Piano#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
            [What are decoder input IDs?](../glossary#decoder-input-ids) Pop2Piano uses the `pad_token_id` as the
            starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
            `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`
        NzSBoth `inputs_embeds` and `input_features` received! Please provide only one of them)r(  r&   rL  r   r   r.   )rN  rW   r  )r(  r&   rL  rÖ   r  r  rP  r-   r"  )Úignore_indexr,   )	ÚlossÚlogitsrÖ   Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_stater  Úencoder_attentions© )r%  rn  rr   r   Úlenr*  rp  ra   Úscale_decoder_outputsrj  r  r   rÇ   r3   r   rÖ   rW   r  r  rN  )rI   r(  r&   r‚  rƒ  r„  rÖ   rL  rv  r…  r†  rP  r*   rW   Údecoder_outputsÚsequence_outputÚ	lm_logitsrŠ  Úloss_fcts                      r<   rY   z)Pop2PianoForConditionalGeneration.forwardS  s  € ðN Ð$¨Ð)CÝÐrÑsÔsÐsØÐ'¨MÐ,AØ*ˆMð Ð"à*˜dœlð Ø#Ø-Ø+ðð ð ð	ð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð '˜$œ,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆàŒ;Ô,ð 	GØ-°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r>   Ú	composer1c                 óŠ  •— |€| j         } |j        d	i |¤Ž t          |d¦  «        st          d¦  «        ‚t	          |j        ¦  «        | j        j        k    r2t          d| j        j        › dt	          |j        ¦  «        › d�¦  «        ‚|                      ||||¬¦  «        \  }} t          ¦   «         j
        d	d|||dœ|¤ŽS )
aŽ  
        Generates token ids for midi outputs.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation
        strategies and code examples, check out the [following guide](./generation_strategies).

        </Tip>

        Parameters:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                This is the featurized version of audio generated by `Pop2PianoFeatureExtractor`.
            attention_mask:
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
            composer (`str`, *optional*, defaults to `"composer1"`):
                This value is passed to `Pop2PianoConcatEmbeddingToMel` to generate different embeddings for each
                `"composer"`. Please make sure that the composer value is present in `composer_to_feature_token` in
                `generation_config`. For an example please see
                https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.json .
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            kwargs:
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
                specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.
        Return:
            [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
                Since Pop2Piano is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
                [`~utils.ModelOutput`] types are:
                    - [`~generation.GenerateEncoderDecoderOutput`],
                    - [`~generation.GenerateBeamEncoderDecoderOutput`]
        Nr{  z¢`composer_to_feature_token` was not found! Please refer to https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.jsonand parse a dict like that.ztconfig.composer_vocab_size must be same as the number of keys in generation_config.composer_to_feature_token! Found z vs ú.)rv  r&   rw  rx  )ÚinputsrL  r&   rx  r�  )rx  rÎ   r  r%  r‘  r{  ra   r^  r�  rC   Úgenerate)rI   rv  r&   rw  rx  r*   rL   s         €r<   r›  z*Pop2PianoForConditionalGeneration.generate¹  s2  ø€ ðl Ð$Ø $Ô 6ÐØ ÐÔ Ð*Ð* 6Ð*Ð*Ð*õ Ð(Ð*EÑFÔFð 	Ýð.ñô ð õ Ð Ô:Ñ;Ô;¸t¼{Ô?^Ò^Ð^ÝðràœÔ8ðrð rå>AÐBSÔBmÑ>nÔ>nðrð rð rñô ð ð *.×)IÒ)IØ)Ø)ØØ/ð	 *Jñ *
ô *
Ñ&ˆ˜ð  �u‰wŒwÔð 
ØØ(Ø)Ø/ð	
ð 
ð
 ð
ð 
ð 	
r>   c                 ó,   — |                       |¦  «        S rq   )r*  )rI   r†  s     r<   Ú%prepare_decoder_input_ids_from_labelszGPop2PianoForConditionalGeneration.prepare_decoder_input_ids_from_labels  s   € Ø× Ò  Ñ(Ô(Ð(r>   rq   )NNNNNNNNNNN)Nr—  N)rZ   r[   r\   Ú_tied_weights_keysr    rD   rt  rH  r4   ÚFloatTensorÚstrr   r�  r   r   Ú
LongTensorÚ
BoolTensorÚtuplers   r
   râ   r   r   r   rY   r6  r›  r�  r]   r^   s   @r<   r  r  ö  sm  ø€ € € € € ð (7Ø'6ðð Ðð
˜ð ð ð ð ð ð ð2ð ð ð:ð :ð :ð 48ð/$ð /$àÔ)ð/$ð ð/$ð ,ð	/$ð
 Ô)¨DÑ0ð/$ð /$ð /$ð /$ðb Øð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø37Ø:>Ø*.Ø!%ðb
ð b
àÔ# dÑ*ðb
ð Ô)¨DÑ0ðb
ð !Ô+¨dÑ2ð	b
ð
 !&Ô 0°4Ñ 7ðb
ð ˜u U¤\Ô2Ô3°dÑ:ðb
ð  ™ðb
ð Ô(¨4Ñ/ðb
ð Ô)¨DÑ0ðb
ð  %Ô0°4Ñ7ðb
ð Ô  4Ñ'ðb
ð ˜$‘;ðb
ð Ð+Ô,ðb
ð 
ðb
ð b
ð b
ñ „^ñ Ôðb
ðH €U„]�_„_ð ØØðW
ð W
ð W
ð W
ð W
ñ „_ðW
ðr)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r>   r  )Nr!   N)Erf  rl  r©   Úcollections.abcr   r4   r   Útorch.nnr   Útransformers.generationr   rK  r   r  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_pop2pianor    Ú
get_loggerrZ   r—   ÚModulers   r¨   r=   r@   r`   rw   r   r‰   rå   rñ   r÷   r  r8  r  r  Ú__all__r�  r>   r<   ú<module>r¶     sÑ  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à 4Ð 4Ð 4Ð 4Ð 4Ð 4à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 4Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð !ØØ)-ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð ”< $Ñ&ð%ð Ð'Ô(ð%ð %ð %ð %ðB+ð +ð +ð +ð +˜œñ +ô +ð +ð4ð ð ð ð ˜RœYñ ô ð ð.ð ð ð ð  "¤)ñ ô ð ð<ð ð ð ð �r”yñ ô ð ð&A8ð A8ð A8ð A8ð A8˜œñ A8ô A8ð A8ðJ=ð =ð =ð =ð = "¤)ñ =ô =ð =ðB9ð 9ð 9ð 9ð 9 2¤9ñ 9ô 9ð 9ð@LPð LPð LPð LPð LPÐ/ñ LPô LPð LPð^ ðP!ð P!ð P!ð P!ð P!˜ñ P!ô P!ñ „ðP!ðfL
ð L
ð L
ð L
ð L
Ð-ñ L
ô L
ð L
ð^ð ð ð ð  B¤Iñ ô ð ð €ððñ ô ð
Y)ð Y)ð Y)ð Y)ð Y)Ð(@À/ñ Y)ô Y)ñô ð
Y)ðx /Ð0JÐ
K€€€r>   