§
    “©ºjh  ã                   óÈ  — d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl2m8Z9 d dl:m;Z;m<Z< d dl=m>Z> d dl?m@Z@mAZAmBZB d dlCmZ d dlDmEZE d dlFmGZGmHZH erd dlImJZJ  ede¬¦  «        ZK G d„ de,¦  «        ZL G d„ de0¦  «        ZM G d „ d!e¦  «        ZN	 d>d"e
d#eOePeeeef         f         d$eQd%eRd&eeOePeePegeQf         f                  d'dfd(„ZS	 	 	 d?d"e
d#eOePeeeef         f         d+eQd,eQd-eeQ         d'eOePef         fd.„ZTd/eRd0eRd1eRd2ejU        d'd3f
d4„ZVd5eWd'ee         fd6„ZXd@d"e
d5ed1eRd+eQd'df
d7„ZY	 dAd9eOePef         d5ed1eRd+eQd'df
d:„ZZd5ed'efd;„Z[d<eOePef         d5ed'eOePef         fd=„Z\dS )Bé    N)Ú	Generator)ÚAbstractContextManagerÚ	ExitStack)Ú	timedelta)ÚPath)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚTypeVarÚUnion)Úrank_zero_only)ÚTensor)ÚModule)Ú	Optimizer)Ú	TypeGuardÚoverride)ÚCheckpointIO)Údefault_pg_timeout)Ú_distributed_checkpoint_loadÚ_distributed_checkpoint_saveÚ_get_full_state_dict_contextÚ_is_full_checkpointÚ_is_sharded_checkpoint)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú
TBroadcastÚ_apply_filterÚ_BackwardSyncControlÚ!_validate_keys_for_strict_loading)ÚReduceOpÚ_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_TORCH_GREATER_EQUAL_2_3Ú_TORCH_GREATER_EQUAL_2_4)Ú_materialize_distributed_module)Ú_METADATA_FILENAMEÚ
_lazy_loadÚ_move_state_into)Ú
reset_seed)Ú_PATHÚ	_Stateful)Ú
DeviceMeshÚTModel)Úboundc                   ó¦  ‡ — e Zd ZdZddddefdeedgef         deed         e	f         deed         e	f         d	e
d
ee         dee         ddfˆ fd„Zed7d„¦   «         Zeedefd„¦   «         ¦   «         Zej        ededdfd„¦   «         ¦   «         Zeedej        fd„¦   «         ¦   «         Zede	fd„¦   «         Zej        de	ddfd„¦   «         Zede	fd„¦   «         Zeedeeef         fd„¦   «         ¦   «         Zedee         fd„¦   «         Zed8d„¦   «         Zed8ˆ fd„¦   «         Zede de fd„¦   «         Z!ede ddfd„¦   «         Z"ed9dee
         de#fd„¦   «         Z$e	 d:d!e%d"ee         d#eee&ef                  de%fd$„¦   «         Z'ed%ed&eddfd'„¦   «         Z(ed;d)e)d*e	de)fd+„¦   «         Z*e	 	 d<d,e+d-eeee e,ef         f         d.ee         d/eeeeeege
f         f                  ddf
d0„¦   «         Z-e	 	 	 d=d,e+d-eee e,eeee e,ef         f         f                  d1e
d2ee
         deeef         f
d3„¦   «         Z.d8d4„Z/defd5„Z0d8d6„Z1ˆ xZ2S )>ÚModelParallelStrategyaß  Enables user-defined parallelism applied to a model.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Currently supports up to 2D parallelism. Specifically, it supports the combination of
    Fully Sharded Data-Parallel 2 (FSDP2) with Tensor Parallelism (DTensor). These PyTorch APIs are currently still
    experimental in PyTorch. Requires PyTorch 2.4 or newer.

    Arguments:
        parallelize_fn: A function that applies parallelisms to a module. The strategy will provide the
            model and device mesh as input.
        data_parallel_size: The number of devices within a data-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of nodes in the cluster.
        tensor_parallel_size: The number of devices within a tensor-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of GPUs in a single node.
        save_distributed_checkpoint: If ``True``, each rank saves its shard of weights and optimizer states to a file.
            The checkpoint is a folder with as many files as the world size.
            If ``False``, the full weights and optimizer states get assembled on rank 0 and saved to a single file.

    ÚautoTNÚparallelize_fnr2   Údata_parallel_sizeÚtensor_parallel_sizeÚsave_distributed_checkpointÚprocess_group_backendÚtimeoutÚreturnc                 ó4  •— t          ¦   «                              ¦   «          t          s$t          t	          | ¦  «        j        › d�¦  «        ‚|| _        || _        || _        d| _	        || _
        || _        || _        t          ¦   «         | _        d | _        d S )Nz  requires PyTorch 2.4 or higher.é   )ÚsuperÚ__init__r*   ÚImportErrorÚtypeÚ__name__Ú_parallelize_fnÚ_data_parallel_sizeÚ_tensor_parallel_sizeÚ
_num_nodesÚ_save_distributed_checkpointÚ_process_group_backendÚ_timeoutÚ_ParallelBackwardSyncControlÚ_backward_sync_controlÚ_device_mesh)Úselfr8   r9   r:   r;   r<   r=   Ú	__class__s          €úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/strategies/model_parallel.pyrB   zModelParallelStrategy.__init__Y   s–   ø€ õ 	‰Œ×ÒÑÔÐÝ'ð 	XÝ¥ d¡¤Ô!4ÐVÐVÐVÑWÔWÐWØ-ˆÔØ#5ˆÔ Ø%9ˆÔ"ØˆŒØ,GˆÔ)Ø5JˆÔ#Ø-4ˆŒÝ&BÑ&DÔ&DˆÔ#à26ˆÔÐÐó    c                 ó<   — | j         €t          d¦  «        ‚| j         S )NzKAccessing the device mesh before processes have initialized is not allowed.)rO   ÚRuntimeError©rP   s    rR   Údevice_meshz!ModelParallelStrategy.device_meshp   s#   € àÔÐ$ÝÐlÑmÔmÐmØÔ Ð rS   c                 óL   — t          dt          | ¦  «        j        › d�¦  «        ‚)NúThe `z3` does not use the `CheckpointIO` plugin interface.©ÚNotImplementedErrorrD   rE   rV   s    rR   Úcheckpoint_ioz#ModelParallelStrategy.checkpoint_iov   ó'   € õ "Ð"r­$¨t©*¬*Ô*=Ð"rÐ"rÐ"rÑsÔsÐsrS   Úioc                 óL   — t          dt          | ¦  «        j        › d�¦  «        ‚)NrY   z3` does not support setting a `CheckpointIO` plugin.rZ   )rP   r^   s     rR   r\   z#ModelParallelStrategy.checkpoint_io{   r]   rS   c                 ó8   — | j         €J ‚| j         | j                 S ©N)Úparallel_devicesÚ
local_rankrV   s    rR   Úroot_devicez!ModelParallelStrategy.root_device€   s#   € ð Ô$Ð0Ð0Ð0ØÔ$ T¤_Ô5Ð5rS   c                 ó   — | j         S ra   ©rI   rV   s    rR   Ú	num_nodeszModelParallelStrategy.num_nodes†   s
   € àŒÐrS   rg   c                 ó   — || _         d S ra   rf   )rP   rg   s     rR   rg   zModelParallelStrategy.num_nodesŠ   s   € à#ˆŒˆˆrS   c                 ó<   — | j         �t          | j         ¦  «        ndS )Nr   )rb   ÚlenrV   s    rR   Únum_processesz#ModelParallelStrategy.num_processesŽ   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUrS   c                 ó€   — | j         €J ‚| j         d         }|                     ¦   «         |                     ¦   «         dœS )NÚdata_parallel)Únum_replicasÚrank)rW   ÚsizeÚget_local_rank)rP   Údata_parallel_meshs     rR   Údistributed_sampler_kwargsz0ModelParallelStrategy.distributed_sampler_kwargs’   sH   € ð ÔÐ+Ð+Ð+Ø!Ô-¨oÔ>ÐØ 2× 7Ò 7Ñ 9Ô 9ÐCU×CdÒCdÑCfÔCfÐgÐgÐgrS   c                 ó   — | j         S ra   )rK   rV   s    rR   r<   z+ModelParallelStrategy.process_group_backend™   s   € àÔ*Ð*rS   c                 ó~   — | j         €J ‚| j         j        s't          | j         | j        | j        ¦  «        | _        d S d S ra   )Úcluster_environmentÚcreates_processes_externallyr   rk   rg   Ú	_launcherrV   s    rR   Ú_configure_launcherz)ModelParallelStrategy._configure_launcher�   sN   € àÔ'Ð3Ð3Ð3ØÔ'ÔDð 	uÝ6°tÔ7OÐQUÔQcÐeiÔesÑtÔtˆDŒNˆNˆNð	uð 	urS   c                 ó"  •— t          ¦   «                              ¦   «          |                      ¦   «          | j        dk    r| j        | _        | j        dk    r| j        | _        t          | j        | j        | j        | j	        ¦  «        | _
        d S )Nr7   )rA   Úsetup_environmentÚ_setup_distributedrG   rg   rH   rk   Ú_setup_device_meshÚ
world_sizerd   rO   )rP   rQ   s    €rR   r{   z'ModelParallelStrategy.setup_environment£   s‰   ø€ å‰Œ×!Ò!Ñ#Ô#Ð#Ø×ÒÑ!Ô!Ð!ØÔ# vÒ-Ð-Ø'+¤~ˆDÔ$ØÔ%¨Ò/Ð/Ø)-Ô);ˆDÔ&Ý.ØÔ$ dÔ&@À$Ä/ÐSWÔScñ
ô 
ˆÔÐÐrS   Úmodulec                 óz  ‡— ddl mŠ t          ˆfd„|                     ¦   «         D ¦   «         ¦  «        rt	          d| j        j        › d�¦  «        ‚|                      || j        ¦  «        }t          |t          ¦  «        s$t	          dt          |¦  «        j        › �¦  «        ‚t          || j        ¦  «         |S )Nr   ©ÚFullyShardedDataParallelc              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S ra   ©Ú
isinstance)Ú.0Úmodr‚   s     €rR   ú	<genexpr>z5ModelParallelStrategy.setup_module.<locals>.<genexpr>³   s.   øè è € ÐUÐU¸S�z˜#Ð7Ñ8Ô8ÐUÐUÐUÐUÐUÐUrS   z\Found modules that are wrapped with `torch.distributed.fsdp.FullyShardedDataParallel`. The `z5` only supports the new FSDP2 APIs in PyTorch >= 2.4.zBThe `parallelize_fn` must return a `nn.Module` instance, but got: )Útorch.distributed.fsdpr‚   ÚanyÚmodulesÚ	TypeErrorrQ   rE   rF   rW   r…   r   rD   r+   rd   )rP   r   r‚   s     @rR   Úsetup_modulez"ModelParallelStrategy.setup_module¯   sæ   ø€ àCÐCÐCÐCÐCÐCåÐUÐUÐUÐUÀFÇNÂNÑDTÔDTÐUÑUÔUÑUÔUð 	ÝðhØœÔ0ðhð hð hñô ð ð
 ×%Ò% f¨dÔ.>Ñ?Ô?ˆÝ˜&¥&Ñ)Ô)ð 	ÝØlÕUYÐZ`ÑUaÔUaÔUjÐlÐlñô ð õ 	(¨°Ô0@ÑAÔAÐAØˆrS   c                 ó   — d S ra   © )rP   r   s     rR   Úmodule_to_devicez&ModelParallelStrategy.module_to_deviceÁ   s   € àˆrS   Ú
empty_initc                 óÐ   — | j                              ¦   «         }t          ¦   «         }|r'|                     t	          j        d¦  «        ¦  «         |                     |¦  «         |S )NÚmeta)Ú	precisionÚmodule_init_contextr   Úenter_contextÚtorchÚdevice)rP   r‘   Úprecision_init_ctxÚstacks       rR   r•   z)ModelParallelStrategy.module_init_contextÅ   sa   € à!œ^×?Ò?ÑAÔAÐÝ‘”ˆØð 	6ð ×Ò¥¤¨VÑ 4Ô 4Ñ5Ô5Ð5Ø×ÒÐ.Ñ/Ô/Ð/ØˆrS   ÚmeanÚtensorr(   Ú	reduce_opc                 óT   — t          |t          ¦  «        rt          |||¬¦  «        S |S )N)r�   )r…   r   r&   )rP   rœ   r(   r�   s       rR   Ú
all_reducez ModelParallelStrategy.all_reduceÐ   s1   € õ �f�fÑ%Ô%ð 	NÝ)¨&°%À9ÐMÑMÔMÐMØˆrS   ÚargsÚkwargsc                 ó   — t          ¦   «         sd S t          j                             ¦   «         dk    r-t          j                             | j        j        g¬¦  «         d S t          j                             ¦   «          d S )NÚnccl)Ú
device_ids)r#   r—   ÚdistributedÚget_backendÚbarrierrd   Úindex)rP   r    r¡   s      rR   r§   zModelParallelStrategy.barrierØ   ss   € å*Ñ,Ô,ð 	ØˆFÝÔ×(Ò(Ñ*Ô*¨fÒ4Ð4ÝÔ×%Ò%°$Ô2BÔ2HÐ1IÐ%ÑJÔJÐJÐJÐJåÔ×%Ò%Ñ'Ô'Ð'Ð'Ð'rS   r   ÚobjÚsrcc                 ó�   — t          ¦   «         s|S |g}t          j                             ||t          j        ¬¦  «         |d         S )Nr'   r   )r#   r—   r¥   Úbroadcast_object_listÚ_groupÚWORLD)rP   r©   rª   s      rR   Ú	broadcastzModelParallelStrategy.broadcastá   sE   € å*Ñ,Ô,ð 	ØˆJàˆeˆÝÔ×/Ò/°°SÅÄÐ/ÑMÔMÐMØ�1ŒvˆrS   ÚpathÚstateÚstorage_optionsÚfilterc                 óZ  — |�:t          dt          | ¦  «        j        › dt          | ¦  «        j        › d�¦  «        ‚|�+| j        r$t	          t          | ¦  «        j        › d�¦  «        ‚t          |                      |¦  «        ¦  «        }t          ||| j         | j        |¬¦  «         dS )aN  Save model, optimizer, and other state to a checkpoint on disk.

        If distributed checkpointing is enabled (default), the checkpoint gets saved as a directory containing one file
        per process, with model- and optimizer shards stored per file. Additionally, it creates a metadata file
        `meta.pt` with the rest of the user's state (only saved from rank 0).
        If distributed checkpointing is disabled (``save_distributed_checkpoint=False``), the checkpoint will be
        written to a single file containing the weights, optimizer state and other metadata.

        Nú`zF.save_checkpoint(..., storage_options=...)` is not supported because `z"` does not use the `CheckpointIO`.zV doesn't support loading distributed filtered checkpoints, so saving them is disabled.)r°   r±   Úfull_state_dictro   r³   )	rŒ   rD   rE   rJ   r[   r   r¯   Ú_save_checkpointÚglobal_rank)rP   r°   r±   r²   r³   s        rR   Úsave_checkpointz%ModelParallelStrategy.save_checkpointê   sè   € ð" Ð&ÝðM•D˜‘J”JÔ'ð Mð MÝ˜$‘Z”ZÔ(ðMð Mð Mñô ð ð Ð $Ô"CÐå%Ý˜‘:”:Ô&ð /ð /ð /ñô ð õ
 �D—N’N 4Ñ(Ô(Ñ)Ô)ˆÝØØØ!%Ô!BÐBØÔ!Øð	
ñ 	
ô 	
ð 	
ð 	
ð 	
rS   ÚstrictÚweights_onlyc           
      ó¼  — |s=t          dt          | ¦  «        j        › d|›dt          | ¦  «        j        › d�¦  «        ‚t          |                      |¦  «        ¦  «        }t          |t          ¦  «        rt          ||| j        |¬¦  «         i S t          |t          ¦  «        r%t          dt          | ¦  «        j        › d�¦  «        ‚t          ||||¬¦  «        S )	zOLoad the contents from a checkpoint and restore the state of the given objects.zGot z.load_checkpoint(..., state=zY) but a state with at least  a model instance to reload is required. Pass it in like so: z2.load_checkpoint(..., state={'model': model, ...}))r   r~   rº   zNLoading a single optimizer object from a checkpoint is not supported yet with ú.)r°   r±   rº   r»   )Ú
ValueErrorrD   rE   r   r¯   r…   r   Ú _load_raw_module_state_from_pathr~   r   r[   Ú_load_checkpoint)rP   r°   r±   rº   r»   s        rR   Úload_checkpointz%ModelParallelStrategy.load_checkpoint  s  € ð ð 	Ýð^•t˜D‘z”zÔ*ð ^ð ^Èð ^ð ^å˜‘J”JÔ'ð^ð ^ð ^ñô ð õ �D—N’N 4Ñ(Ô(Ñ)Ô)ˆå�e�VÑ$Ô$ð 	Ý,¨T¸%ÈDÌOÐdjÐkÑkÔkÐkØˆIå�e�YÑ'Ô'ð 	Ý%ØwÕaeÐfjÑakÔakÔatÐwÐwÐwñô ð õ   T°¸vÐT`ÐaÑaÔaÐarS   c                 ó  — t          ¦   «          |                      ¦   «          |                      ¦   «         | _        | j        €J ‚d| j        i}t          r| j        j        dk    r| j        nd |d<   t          | j        | j        fi |¤Ž d S )Nr=   ÚcpuÚ	device_id)
r/   Ú_set_world_ranksÚ_get_process_group_backendrK   rv   rL   r)   rd   rD   r%   )rP   r¡   s     rR   r|   z(ModelParallelStrategy._setup_distributed-  s�   € Ý‰ŒˆØ×ÒÑÔÐØ&*×&EÒ&EÑ&GÔ&GˆÔ#ØÔ'Ð3Ð3Ð3Ø"+¨T¬]Ð!;ˆÝ#ð 	_Ø6:Ô6FÔ6KÈuÒ6TÐ6T $Ô"2Ð"2ÐZ^ˆF�;ÑÝ˜dÔ6¸Ô8SÐ^Ð^ÐW]Ð^Ð^Ð^Ð^Ð^rS   c                 ó8   — | j         pt          | j        ¦  «        S ra   )rK   r$   rd   rV   s    rR   rÆ   z0ModelParallelStrategy._get_process_group_backend7  s   € ØÔ*ÐmÕ.[Ð\`Ô\lÑ.mÔ.mÐmrS   c                 óú   — | j         �V| j                              | j        | j        z  | j        z   ¦  «         | j                              | j        | j        z  ¦  «         | j        xt          _	        t          _	        d S ra   )rv   Úset_global_rankÚ	node_rankrk   rc   Úset_world_sizerg   r¸   r   ro   Úutils_rank_zero_onlyrV   s    rR   rÅ   z&ModelParallelStrategy._set_world_ranks:  sr   € ØÔ#Ð/ØÔ$×4Ò4°T´^ÀdÔFXÑ5XÐ[_Ô[jÑ5jÑkÔkÐkØÔ$×3Ò3°D´NÀTÔEWÑ4WÑXÔXÐXð ;?Ô:JÐJ�ÔÕ2Ô7Ð7Ð7rS   )r>   r2   ©r>   Nra   )Nr›   )r   )NN)NTN)3rE   Ú
__module__Ú__qualname__Ú__doc__r   r
   r3   r   r   ÚintÚboolr   Ústrr   rB   ÚpropertyrW   r   r   r\   Úsetterr—   r˜   rd   rg   rk   Údictr	   rs   r<   ry   r{   r   r�   r�   r   r•   r   r"   rŸ   r§   r   r¯   r0   r   r¹   rÁ   r|   rÆ   rÅ   Ú__classcell__)rQ   s   @rR   r6   r6   C   sÆ  ø€ € € € € ðð ð0 ;AØ<BØ,0Ø/3Ø'9ð7ð 7à  &¨,Ð!7¸Ð!?Ô@ð7ð " '¨&¤/°3Ð"6Ô7ð7ð $ G¨F¤O°SÐ$8Ô9ð	7ð
 &*ð7ð  (¨œ}ð7ð ˜)Ô$ð7ð 
ð7ð 7ð 7ð 7ð 7ð 7ð. ð!ð !ð !ñ „Xð!ð
 Øðt˜|ð tð tð tñ „Xñ „Xðtð ÔØðt ð t°ð tð tð tñ „Xñ Ôðtð Øð6˜Uœ\ð 6ð 6ð 6ñ „Xñ „Xð6ð ð˜3ð ð ð ñ „Xðð Ôð$ 3ð $¨4ð $ð $ð $ñ Ôð$ð ðV˜sð Vð Vð Vñ „XðVð Øðh¨D°°c°¬Nð hð hð hñ „Xñ „Xðhð
 ð+ x°¤}ð +ð +ð +ñ „Xð+ð ðuð uð uñ „Xðuð
 ð	
ð 	
ð 	
ð 	
ð 	
ñ „Xð	
ð ð 6ð ¨fð ð ð ñ „Xðð" ð vð °$ð ð ð ñ „Xðð ðð ¨h°t¬nð ÐH^ð ð ð ñ „Xðð àgmðð ØðØ%-¨c¤]ðØFNÈuÐU]Ð_bÐUbÔOcÔFdðà	ðð ð ñ „Xðð ð(˜Sð (¨Cð (°Dð (ð (ð (ñ „Xð(ð ðð ˜Zð ¨cð ¸*ð ð ð ñ „Xðð ð
 *.ØBFð#
ð #
àð#
ð �C˜˜v y°#Ð5Ô6Ð6Ô7ð#
ð " #œð	#
ð
 ˜˜c 8¨S°#¨J¸Ð,<Ô#=Ð=Ô>Ô?ð#
ð 
ð#
ð #
ð #
ñ „Xð#
ðJ ð _cØØ'+ðbð bàðbð ˜˜f i°°c¸5ÀÈÐTWÐAWÔ;XÐ6XÔ1YÐYÔZÔ[ðbð ð	bð
 ˜t”nðbð 
ˆc�3ˆhŒðbð bð bñ „Xðbð8_ð _ð _ð _ðn¨Cð nð nð nð nðKð Kð Kð Kð Kð Kð Kð KrS   r6   c                   ó2   — e Zd Zedededefd„¦   «         ZdS )rM   r   Úenabledr>   c                 ó$   — t          ||¬¦  «        S )z9Blocks gradient synchronization inside the FSDP2 modules.)r   rÙ   )Ú_FSDPNoSync©rP   r   rÙ   s      rR   Úno_backward_syncz-_ParallelBackwardSyncControl.no_backward_syncD  s   € õ  &°'Ð:Ñ:Ô:Ð:rS   N)rE   rÎ   rÏ   r   r   rÒ   r   rÝ   r�   rS   rR   rM   rM   C  sH   € € € € € Øð; vð ;¸ð ;ÐAWð ;ð ;ð ;ñ „Xð;ð ;ð ;rS   rM   c                   óR   — e Zd Zdededdfd„Zdeddfd„Zdd„Zd	ed
ededdfd„Z	dS )rÛ   r   rÙ   r>   Nc                 ó"   — || _         || _        d S ra   )Ú_moduleÚ_enabledrÜ   s      rR   rB   z_FSDPNoSync.__init__K  s   € ØˆŒØˆŒˆˆrS   Úrequires_grad_syncc                 ó˜   — ddl m} | j                             ¦   «         D ])}t	          ||¦  «        r|                     |d¬¦  «         Œ*d S )Nr   )Ú
FSDPModuleF©Úrecurse)Ú"torch.distributed._composable.fsdprä   rà   r‹   r…   Úset_requires_gradient_sync)rP   râ   rä   r‡   s       rR   Ú_set_requires_grad_syncz#_FSDPNoSync._set_requires_grad_syncO  sm   € ØAÐAÐAÐAÐAÐAà”<×'Ò'Ñ)Ô)ð 	Rð 	RˆCÝ˜#˜zÑ*Ô*ð RØ×.Ò.Ð/AÈ5Ð.ÑQÔQÐQøð	Rð 	RrS   c                 ó<   — |                       | j         ¦  «         d S ra   ©ré   rá   rV   s    rR   Ú	__enter__z_FSDPNoSync.__enter__V  s!   € Ø×$Ò$¨¬Ð%6Ñ7Ô7Ð7Ð7Ð7rS   Úexc_typeÚ	exc_valueÚ	tracebackc                 ó:   — |                       | j        ¦  «         d S ra   rë   )rP   rí   rî   rï   s       rR   Ú__exit__z_FSDPNoSync.__exit__Y  s   € Ø×$Ò$ T¤]Ñ3Ô3Ð3Ð3Ð3rS   rÍ   )
rE   rÎ   rÏ   r   rÒ   rB   ré   rì   r	   rñ   r�   rS   rR   rÛ   rÛ   J  s©   € € € € € ð ˜vð  °ð  ¸ð  ð  ð  ð  ðR¸$ð RÀ4ð Rð Rð Rð Rð8ð 8ð 8ð 8ð4 ð 4°ð 4Àð 4Èð 4ð 4ð 4ð 4ð 4ð 4rS   rÛ   r°   r±   r¶   ro   r³   r>   c                 ód  — |                       ¦   «         r#|r!t          | ¦  «        st          d| › �¦  «        ‚d„ |                     ¦   «         D ¦   «         }t	          |¦  «        dk    rt          d¦  «        ‚t	          |¦  «        dk    rt          d¦  «        ‚|d         }ddlm}m}m	}	  ||d¬	¦  «        }
i }i }| 
                    ¦   «         D ]‘\  }}t          |t          ¦  «        r |||
¬
¦  «        }|}nSt          |t          ¦  «        r |	|||
¬
¦  «        }|}n-t          |t          ¦  «        r|                     ¦   «         n|}|}t!          ||pi ||¦  «         Œ’|rWt          | ¦  «        rt#          j        | ¦  «         |                     |¦  «         |dk    rt)          j        || ¦  «         d S d S |                      ¦   «         r|                      ¦   «          |                      dd¬¦  «         t3          || ¦  «         |dk    rt)          j        || t4          z  ¦  «         d S d S )Nz/The checkpoint path exists and is a directory: c                 ó0   — g | ]}t          |¦  «        ¯|‘ŒS r�   ©Ú_has_dtensor_modules)r†   r   s     rR   ú
<listcomp>z$_save_checkpoint.<locals>.<listcomp>g  s&   € ÐSÐSÐS˜&Õ6JÈ6Ñ6RÔ6RÐSˆvÐSÐSÐSrS   r   a  Could not find a distributed model in the provided checkpoint state. Please provide the model as part of the state like so: `save_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before saving the checkpoint.r@   zêFound multiple distributed models in the given state. Saving distributed checkpoints is currently limited to a single model per checkpoint. To save multiple models, call the save method for each model separately with a different path.)ÚStateDictOptionsÚget_model_state_dictÚget_optimizer_state_dictT)r¶   Úcpu_offload©Úoptions)ÚparentsÚexist_ok)Úis_dirr   ÚIsADirectoryErrorÚvaluesrj   r¾   Ú'torch.distributed.checkpoint.state_dictr÷   rø   rù   Úitemsr…   r   r   r1   Ú
state_dictr   ÚshutilÚrmtreeÚupdater—   ÚsaveÚis_fileÚunlinkÚmkdirr   r,   )r°   r±   r¶   ro   r³   r‹   r   r÷   rø   rù   Ústate_dict_optionsÚconverted_stateÚmetadataÚkeyr©   Ú	convertedÚtarget_dicts                    rR   r·   r·   ]  s˜  € ð ‡{‚{�}„}ð Z˜ð ZÕ1GÈÑ1MÔ1Mð ZÝÐ XÐRVÐ XÐ XÑYÔYÐYàSÐS E§L¢L¡N¤NÐSÑSÔS€GÝ
ˆ7�|„|�qÒÐÝðoñ
ô 
ð 	
õ
 ˆ7�|„|�aÒÐÝðLñ
ô 
ð 	
ð
 �QŒZ€FàxÐxÐxÐxÐxÐxÐxÐxÐxÐxà)Ð)¸/ÐW[Ð\Ñ\Ô\Ðð ')€OØ!€HØ—K’K‘M”Mð Að A‰ˆˆSå�c�6Ñ"Ô"ð 	#Ø,Ð,¨SÐ:LÐMÑMÔMˆIØ)ˆKˆKÝ˜�YÑ'Ô'ð 	#Ø0Ð0°¸ÐFXÐYÑYÔYˆIØ)ˆKˆKå,6°s½IÑ,FÔ,FÐO˜ŸšÑ(Ô(Ð(ÈCˆIØ"ˆKÝ�c˜6˜< R¨°KÑ@Ô@Ð@Ð@àð <Ý! $Ñ'Ô'ð 	 ÝŒM˜$ÑÔÐØ×Ò˜xÑ(Ô(Ð(Ø�1Š9ˆ9ÝŒJ�¨Ñ-Ô-Ð-Ð-Ð-ð ˆ9ð �<Š<‰>Œ>ð 	Ø�KŠK‰MŒMˆMØ�
Š
˜4¨$ˆ
Ñ/Ô/Ð/Ý$ _°dÑ;Ô;Ð;Ø�1Š9ˆ9ÝŒJ�x Õ(:Ñ!:Ñ;Ô;Ð;Ð;Ð;ð ˆ9rS   TFrº   Úoptimizer_states_from_listr»   c                 ó¼  — ddl m}m}m}m} d„ |                     ¦   «         D ¦   «         }	t          |	¦  «        dk    rt          d¦  «        ‚d„ |                     ¦   «         D ¦   «         }
t          |	¦  «        dk    rt          d¦  «        ‚t          |	                     ¦   «         ¦  «        d         \  }}t          | ¦  «        �r7 |d¬	¦  «        }| ||¦  «        i}t          || ¦  «         |                     ||         |¬
¦  «         |
                     ¦   «         D ]8\  }}| |||¦  «        i}t          || ¦  «          |||||         |¬¦  «         Œ9t          j        | t          z  |¬¦  «        }|                     ¦   «         |	                     ¦   «         z
  |
                     ¦   «         z
  }t!          ||                     ¦   «         |¬
¦  «         |D ]}||vrŒ|                     |¦  «        ||<   Œ |S t%          | ¦  «        �r0t          j        | dd|¬¦  «        }t'          |                     |¦  «        ||¬
¦  «          |dd|¬¦  «        }t)          |
                     ¦   «         ¦  «        D ]M\  }\  }}|r|d         |         }n|                     |¦  «        }t+          ||¦  «        } |||||¬¦  «         ŒN|                     ¦   «         |	                     ¦   «         z
  |
                     ¦   «         z
  }t!          ||                     ¦   «         |¬
¦  «         t-          |||¬¦  «         |S t          dt/          | ¦  «        ›d�¦  «        ‚)Nr   )r÷   rø   rù   Úset_optimizer_state_dictc                 ó8   — i | ]\  }}t          |¦  «        ¯||“ŒS r�   rô   )r†   r  r   s      rR   ú
<dictcomp>z$_load_checkpoint.<locals>.<dictcomp>¨  s-   € Ð\Ð\Ð\™{˜s FÕ?SÐTZÑ?[Ô?[Ð\ˆs�FÐ\Ð\Ð\rS   a  Could not find a distributed model in the provided checkpoint state. Please provide the model as part of the state like so: `load_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before loading the checkpoint.c                 óD   — i | ]\  }}t          |t          ¦  «        ¯||“ŒS r�   )r…   r   )r†   r  Úoptims      rR   r  z$_load_checkpoint.<locals>.<dictcomp>¯  s.   € Ð]Ð]Ð]¡  eÅ
È5ÕR[Ñ@\Ô@\Ð]�#�uÐ]Ð]Ð]rS   r@   zëFound multiple distributed models in the given state. Loading distributed checkpoints is currently limited to a single model per checkpoint. To load multiple models, call the load method for each model separately with a different path.T)rú   ©rº   )Úoptim_state_dictrü   )r»   rÃ   )ÚmmapÚmap_locationr»   ©Úbroadcast_from_rank0r¶   rº   Úoptimizer_states)ÚsourceÚdestinationÚkeysz	The path z£ does not point to a valid checkpoint. Make sure the path points to either a directory with distributed checkpoint shards, or a single file with a full checkpoint.)r  r÷   rø   rù   r  r  rj   r¾   Úlistr   r   Úload_state_dictr—   Úloadr,   r"  r!   Úpopr   Ú_load_raw_module_stateÚ	enumerateÚ _rekey_optimizer_state_if_neededr.   rÓ   )r°   r±   rº   r  r»   r÷   rø   rù   r  r‹   Ú
optimizersÚ
module_keyr   r  Úmodule_stateÚ	optim_keyr  Úoptim_stater  Úrequested_metadata_keysr  Ú
checkpointÚoptimizer_idxÚoptimizer_nameÚ	optimizerÚoptimizer_states                             rR   rÀ   rÀ   š  s  € ðð ð ð ð ð ð ð ð ð ð ð ð ]Ð\¨e¯kªk©m¬mÐ\Ñ\Ô\€GÝ
ˆ7�|„|�qÒÐÝðpñ
ô 
ð 	
ð
 ^Ð]¨u¯{ª{©}¬}Ð]Ñ]Ô]€JÝ
ˆ7�|„|�aÒÐÝðLñ
ô 
ð 	
õ
 ˜gŸmšm™oœoÑ.Ô.¨qÔ1Ñ€J�å˜dÑ#Ô#ñ Ø-Ð-¸$Ð?Ñ?Ô?Ðà"Ð$8Ð$8¸Ñ$@Ô$@ÐAˆÝ$ \°4Ñ8Ô8Ð8Ø×Ò˜|¨JÔ7ÀÐÑGÔGÐGð !+× 0Ò 0Ñ 2Ô 2ð 	yð 	yÑˆI�uØ$Ð&>Ð&>¸vÀuÑ&MÔ&MÐNˆKÝ(¨°dÑ;Ô;Ð;Ø$Ð$ V¨UÀ[ÐQZÔE[ÐewÐxÑxÔxÐxÐxõ ”:˜dÕ%7Ñ7ÀlÐSÑSÔSˆØ"'§*¢*¡,¤,°·²±´Ñ"?À*Ç/Â/ÑBSÔBSÑ"SÐÝ)Ð*AÀ8Ç=Â=Á?Ä?Ð[aÐbÑbÔbÐbØ*ð 	+ð 	+ˆCØ˜(Ð"Ð"ØØ!Ÿš cÑ*Ô*ˆE�#‰JˆJð ˆå˜4Ñ Ô ñ  Ý”Z ¨4¸eÐR^Ð_Ñ_Ô_ˆ
Ý˜zŸ~š~¨jÑ9Ô9¸6È&ÐQÑQÔQÐQà-Ð-Ø!%Ø Øð
ñ 
ô 
Ðõ
 ;DÀJ×DTÒDTÑDVÔDVÑ:WÔ:Wð 	ð 	Ñ6ˆMÑ6˜N¨IØ)ð Að #-Ð-?Ô"@ÀÔ"O��à",§.¢.°Ñ"@Ô"@�å>¸ÐPVÑWÔWˆOØ$Ð$ØØØ!0Ø*ð	ñ ô ð ð ð #(§*¢*¡,¤,°·²±´Ñ"?À*Ç/Â/ÑBSÔBSÑ"SÐÝ)Ð*AÀ:Ç?Â?ÑCTÔCTÐ]cÐdÑdÔdÐdõ 	 
¸ÐD[Ð\Ñ\Ô\Ð\ð Ðå
ð	b•C˜‘I”Ið 	bð 	bð 	bñô ð rS   r9   r:   r~   r˜   r2   c           	      ó|   — ddl m} | |z  |k    rt          d| › d|› d|› d�¦  «        ‚ ||j        | |fd¬¦  «        S )	Nr   )Úinit_device_meshzThe sizes `data_parallel_size=z` and `tensor_parallel_size=z*` multiplied should equal the world size (z).)rm   Útensor_parallel)Údevice_typeÚ
mesh_shapeÚmesh_dim_names)Útorch.distributed.device_meshr6  rU   rD   )r9   r:   r~   r˜   r6  s        rR   r}   r}   ù  s—   € ð ?Ð>Ð>Ð>Ð>Ð>àÐ0Ñ0°JÒ>Ð>Ýð Ð-?ð  ð  Ø&:ð ð  àð ð  ð  ñ
ô 
ð 	
ð
 ÐØ”KØ&Ð(<Ð=Ø;ðñ ô ð rS   r   c                 ó”   ‡— ddl mŠ t          | t          ¦  «        o,t	          ˆfd„|                      ¦   «         D ¦   «         ¦  «        S )Nr   )ÚDTensorc              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S ra   r„   )r†   Útr=  s     €rR   rˆ   z'_has_dtensor_modules.<locals>.<genexpr>  s-   øè è € Ð-bÐ-bÈ­j¸¸GÑ.DÔ.DÐ-bÐ-bÐ-bÐ-bÐ-bÐ-brS   )Útorch.distributed._tensorr=  r…   r   rŠ   Ú
parameters)r   r=  s    @rR   rõ   rõ     sS   ø€ Ø1Ð1Ð1Ð1Ð1Ð1å�f�fÑ%Ô%Ðb­#Ð-bÐ-bÐ-bÐ-bÈf×N_ÒN_ÑNaÔNaÐ-bÑ-bÔ-bÑ*bÔ*bÐbrS   c                 óÈ   — t          | ¦  «        st          d| › �¦  «        ‚t          rt          j        | dd¬¦  «        nt          | ¦  «        }t          ||||¬¦  «         dS )z;Loads the state dict from a file path into the FSDP module.zxFailed to load checkpoint directly into the model. The given path must be a single file containing the full state dict: TrÃ   )r  r  )r  r   r~   rº   N)r   r¾   r)   r—   r%  r-   r'  )r°   r   r~   rº   r  s        rR   r¿   r¿     s‚   € å˜tÑ$Ô$ð 
Ýð(Ø!%ð(ð (ñ
ô 
ð 	
õ
 E]Ðr•”˜D t¸%Ð@Ñ@Ô@Ð@ÕblÐmqÑbrÔbr€JÝ j¸ÈJÐ_eÐfÑfÔfÐfÐfÐfrS   r@   r  c                 ó  — ddl m} t          |¦  «        r…ddlm}m}  |ddd¬¦  «        }|                     ¦   «         D ]X\  }}	t          |	¦  «        D ]C\  }
}|› |rdnd› |
› �}|| vr|sŒt          d	|› d
�¦  «        ‚|
| |         i} ||	||¬¦  «         ŒDŒYdS t          ||¦  «        rDt          ||d¬¦  «        5  |                     | |¬¦  «         ddd¦  «         dS # 1 swxY w Y   dS |                     | |¬¦  «         dS )zlLoads the state dict into the module by gathering all weights first and then and writing back to each shard.r   r�   )r÷   Úset_model_state_dictTFr  r½   Ú zThe model contains a key 'z^' that does not exist in the loaded checkpoint. To disable strict loading, set `strict=False`.rû   )r~   Ú
rank0_onlyr  N)r‰   r‚   rõ   r  r÷   rD  Únamed_modulesÚ%_named_parameters_and_buffers_to_loadÚKeyErrorr…   r   r$  )r  r   r~   rº   ÚFSDPr÷   rD  r  Úsubmodule_nameÚ	submoduleÚ
param_nameÚ_Úfull_param_nameÚlocal_state_dicts                 rR   r'  r'     s  € ð HÐGÐGÐGÐGÐGå˜FÑ#Ô#ð :ØbÐbÐbÐbÐbÐbÐbÐbà-Ð-Ø!%Ø àð	
ñ 
ô 
Ðð *0×)=Ò)=Ñ)?Ô)?ð 	^ð 	^Ñ%ˆN˜IÝ!FÀyÑ!QÔ!Qð 
^ð 
^‘�
˜AØ%3Ð"`¸NÐ5R°S°SÐPRÐ"`ÐT^Ð"`Ð"`�Ø"¨*Ð4Ð4Ø!ð !Ø Ý"ðJ°_ð Jð Jð Jñô ð ð %/°
¸?Ô0KÐ#LÐ Ø$Ð$ YÐ0@ÐJ\Ð]Ñ]Ô]Ð]Ð]ð
^ð	^ð 	^õ 
�F˜DÑ	!Ô	!ð :Ý)¨&¸ZÐTYÐZÑZÔZð 	>ð 	>Ø×"Ò" :°fÐ"Ñ=Ô=Ð=ð	>ð 	>ð 	>ñ 	>ô 	>ð 	>ð 	>ð 	>ð 	>ð 	>ð 	>ð 	>øøøð 	>ð 	>ð 	>ð 	>ð 	>ð 	>ð 	×Ò˜z°&ÐÑ9Ô9Ð9Ð9Ð9s   Â=C"Ã"C&Ã)C&c              #   ó°   K  — t          j        |                      d¬¦  «        |                      d¬¦  «        ¦  «        D ]\  }}|| j        v rŒ||fV — ŒdS )zEReturns parameters and buffers, with non-persistent buffers excluded.Frå   N)Ú	itertoolsÚchainÚnamed_buffersÚnamed_parametersÚ_non_persistent_buffers_set)r   rM  Úparams      rR   rH  rH  D  s   è è € å&œ_Ø×Ò UÐÑ+Ô+Ø×Ò¨ÐÑ.Ô.ñô ð  ð  Ñˆ
�Eð ˜Ô;Ð;Ð;ØØ˜%ÐÐÐÐÐð ð  rS   Úoptimizer_state_dictc                 óÖ   — ddl m} ddl m} t          t	          | d                              ¦   «         ¦  «        d         t          ¦  «        r|                     | |j        |¦  «        } | S )zyHandles the case where the optimizer state is saved from a normal optimizer and converts the keys to parameter
    names.r   r�   )ÚOptimStateKeyTyper±   )	r‰   r‚   rZ  r…   r#  r"  rÑ   Úrekey_optim_state_dictÚ
PARAM_NAME)rX  r   rJ  rZ  s       rR   r)  r)  O  s   € ð HÐGÐGÐGÐGÐGØ8Ð8Ð8Ð8Ð8Ð8å•$Ð+¨GÔ4×9Ò9Ñ;Ô;Ñ<Ô<¸QÔ?ÅÑEÔEð wØ#×:Ò:Ð;OÐQbÔQmÐouÑvÔvÐØÐrS   ra   )TFN)T)r@   T)]rR  r  Úcollections.abcr   Ú
contextlibr   r   Údatetimer   Úpathlibr   Útypingr   r	   r
   r   r   r   r   r—   Ú"lightning_utilities.core.rank_zeror   rÌ   r   Útorch.nnr   Útorch.optimr   Útyping_extensionsr   r   Úlightning_fabric.pluginsr   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Ú lightning_fabric.strategies.fsdpr   r   r   r   r   Ú7lightning_fabric.strategies.launchers.subprocess_scriptr   Ú$lightning_fabric.strategies.parallelr   Ú$lightning_fabric.strategies.strategyr   r   r    r!   Ú&lightning_fabric.utilities.distributedr"   r#   r$   r%   r&   r(   r­   Ú"lightning_fabric.utilities.importsr)   r*   Úlightning_fabric.utilities.initr+   Úlightning_fabric.utilities.loadr,   r-   r.   Ú$lightning_fabric.utilities.rank_zeroÚlightning_fabric.utilities.seedr/   Ú lightning_fabric.utilities.typesr0   r1   r;  r2   r3   r6   rM   rÛ   rÖ   rÓ   rÒ   rÑ   r·   rÀ   r˜   r}   Úobjectrõ   r¿   r'  rH  r)  r�   rS   rR   ú<module>rt     sÿ  ðð Ð Ð Ð Ø €€€Ø %Ð %Ð %Ð %Ð %Ð %Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ Rà €€€Ø UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1à 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø TÐ TÐ TÐ TÐ TÐ Tðð ð ð ð ð ð ð ð ð ð ð ð ð ð ^Ð ]Ð ]Ð ]Ð ]Ð ]Ø AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ KÐ KÐ KÐ KÐ KÐ KØ \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =àð 9Ø8Ð8Ð8Ð8Ð8Ð8à	ˆ� Ð	(Ñ	(Ô	(€ð}Kð }Kð }Kð }Kð }KÐ,ñ }Kô }Kð }Kð@;ð ;ð ;ð ;ð ;Ð#7ñ ;ô ;ð ;ð4ð 4ð 4ð 4ð 4Ð(ñ 4ô 4ð 4ð0 ?Cð:<ð :<Ø
ð:<à��U˜6 9¨cÐ1Ô2Ð2Ô3ð:<ð ð:<ð ð	:<ð
 �T˜#˜x¨¨c¨
°DÐ(8Ô9Ð9Ô:Ô;ð:<ð 
ð:<ð :<ð :<ð :<ð@ Ø',Ø#'ð\ð \Ø
ð\à��U˜6 9¨cÐ1Ô2Ð2Ô3ð\ð ð\ð !%ð	\ð
 ˜4”.ð\ð 
ˆ#ˆsˆ(„^ð\ð \ð \ð \ð~Øðàðð ðð ŒLð	ð
 ðð ð ð ð*c ð c¨I°fÔ,=ð cð cð cð cð	gð 	g¨4ð 	g¸ð 	gÈSð 	gÐZ^ð 	gÐjnð 	gð 	gð 	gð 	gð UYð!:ð !:Ø�S˜#�X”ð!:Ø(.ð!:Ø<?ð!:ØMQð!:à	ð!:ð !:ð !:ð !:ðH °&ð  ¸Yð  ð  ð  ð  ð ¸4ÀÀSÀ¼>ð  ÐSYð  Ð^bÐcfÐhkÐckÔ^lð  ð  ð  ð  ð  ð  rS   