§
    “©ºj+  ã                   ó~  — d dl mZmZ d dlmZ d dlmZmZmZm	Z	 d dl
Z
d dlZ
d dlmZ d dl
mZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0 d dl+m1Z2 d dl3m4Z4 d dl5mZ dZ6 G d„ de%¦  «        Z7 G d„ de*¦  «        Z8dS )é    )ÚAbstractContextManagerÚnullcontext)Ú	timedelta)ÚAnyÚLiteralÚOptionalÚUnionN)Úrank_zero_only)ÚTensor)ÚModule)ÚDistributedDataParallel)Úoverride)ÚAccelerator)Údefault_pg_timeout)ÚClusterEnvironment)ÚCheckpointIO)Ú	Precision)Ú_MultiProcessingLauncher)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú_StrategyRegistry)Ú
TBroadcastÚ_BackwardSyncControl)ÚReduceOpÚ_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_TORCH_GREATER_EQUAL_2_3)Úddp_forkÚddp_notebookc                   óæ  ‡ — e Zd ZdZddddddedfdee         deeej	                          dee
         dee         dee         d	ee         d
ee         ded         deddfˆ fd„Zeedej	        fd„¦   «         ¦   «         Zedefd„¦   «         Zej        deddfd„¦   «         Zedefd„¦   «         Zeedeeef         fd„¦   «         ¦   «         Zedee         fd„¦   «         Zed2d„¦   «         Zed2ˆ fd„¦   «         Zededefd„¦   «         Z ededdfd„¦   «         Z!e	 d3de"dee         dee#e$ef                  de"fd „¦   «         Z%ed!ededdfd"„¦   «         Z&ed4d$e'd%ede'fd&„¦   «         Z(ededeee#ee"f         f         fˆ fd'„¦   «         Z)e	 d5ded)eee#ee"f         f         d*e*ddfˆ fd+„¦   «         Z+e,ed,e-ddfd-„¦   «         ¦   «         Z.d2d.„Z/defd/„Z0d2d0„Z1deee                  fd1„Z2ˆ xZ3S )6ÚDDPStrategyzKStrategy for multi-process single-device training on one or multiple nodes.NÚpopenÚacceleratorÚparallel_devicesÚcluster_environmentÚcheckpoint_ioÚ	precisionÚprocess_group_backendÚtimeoutÚstart_method)r&   ÚspawnÚforkÚ
forkserverÚkwargsÚreturnc	                 óÀ   •— t          ¦   «                              |||||¬¦  «         d| _        || _        || _        || _        t          ¦   «         | _        |	| _        d S )N)r'   r(   r)   r*   r+   é   )	ÚsuperÚ__init__Ú
_num_nodesÚ_process_group_backendÚ_timeoutÚ_start_methodÚ_DDPBackwardSyncControlÚ_backward_sync_controlÚ_ddp_kwargs)Úselfr'   r(   r)   r*   r+   r,   r-   r.   r2   Ú	__class__s             €úo/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/strategies/ddp.pyr7   zDDPStrategy.__init__8   sq   ø€ õ 	‰Œ×ÒØ#Ø-Ø 3Ø'Øð 	ñ 	
ô 	
ð 	
ð ˆŒØ5JˆÔ#Ø-4ˆŒØ)ˆÔÝ&=Ñ&?Ô&?ˆÔ#Ø!ˆÔÐÐó    c                 ó8   — | j         €J ‚| j         | j                 S ©N)r(   Ú
local_rank©r?   s    rA   Úroot_devicezDDPStrategy.root_deviceR   s#   € ð Ô$Ð0Ð0Ð0ØÔ$ T¤_Ô5Ð5rB   c                 ó   — | j         S rD   ©r8   rF   s    rA   Ú	num_nodeszDDPStrategy.num_nodesX   s
   € àŒÐrB   rJ   c                 ó   — || _         d S rD   rI   )r?   rJ   s     rA   rJ   zDDPStrategy.num_nodes\   s   € ð $ˆŒˆˆrB   c                 ó<   — | j         �t          | j         ¦  «        ndS )Nr   )r(   ÚlenrF   s    rA   Únum_processeszDDPStrategy.num_processesa   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUrB   c                 ó0   — | j         | j        z  | j        dœS )N)Únum_replicasÚrank)rJ   rN   Úglobal_rankrF   s    rA   Údistributed_sampler_kwargsz&DDPStrategy.distributed_sampler_kwargse   s   € ð "&¤°$Ô2DÑ!DÈtÔO_Ð`Ð`Ð`rB   c                 ó   — | j         S rD   )r9   rF   s    rA   r,   z!DDPStrategy.process_group_backendj   s   € àÔ*Ð*rB   c                 ó²   — | j         €J ‚| j        dk    r't          | j         | j        | j        ¦  «        | _        d S t          | | j        ¬¦  «        | _        d S )Nr&   )r.   )r)   r;   r   rN   rJ   Ú	_launcherr   rF   s    rA   Ú_configure_launcherzDDPStrategy._configure_launchern   s\   € àÔ'Ð3Ð3Ð3ØÔ Ò(Ð(Ý6°tÔ7OÐQUÔQcÐeiÔesÑtÔtˆDŒNˆNˆNå5°dÈÔI[Ð\Ñ\Ô\ˆDŒNˆNˆNrB   c                 óp   •— t          ¦   «                              ¦   «          |                      ¦   «          d S rD   )r6   Úsetup_environmentÚ_setup_distributed)r?   r@   s    €rA   rY   zDDPStrategy.setup_environmentv   s1   ø€ å‰Œ×!Ò!Ñ#Ô#Ð#Ø×ÒÑ!Ô!Ð!Ð!Ð!rB   Úmodulec                 ó   — |                       ¦   «         }|�;t          j                             t          j                             ¦   «         ¦  «        nt          ¦   «         }|5  t          d||dœ| j        ¤Žcddd¦  «         S # 1 swxY w Y   dS )z^Wraps the model into a :class:`~torch.nn.parallel.distributed.DistributedDataParallel` module.N)r[   Ú
device_ids© )Ú_determine_ddp_device_idsÚtorchÚcudaÚstreamÚStreamr   r   r>   )r?   r[   r]   Úctxs       rA   Úsetup_modulezDDPStrategy.setup_module{   sã   € ð ×3Ò3Ñ5Ô5ˆ
à8BÐ8N�eŒj×Ò¥¤
× 1Ò 1Ñ 3Ô 3Ñ4Ô4Ð4ÕT_ÑTaÔTaˆØð 	eð 	eÝ*Ðd°&ÀZÐdÐdÐSWÔScÐdÐdð	eð 	eð 	eð 	eñ 	eô 	eð 	eð 	eð 	eð 	eð 	eð 	eøøøð 	eð 	eð 	eð 	eð 	eð 	es   Á"BÂBÂ
Bc                 ó:   — |                      | j        ¦  «         d S rD   )ÚtorG   )r?   r[   s     rA   Úmodule_to_devicezDDPStrategy.module_to_device„   s   € à�	Š	�$Ô"Ñ#Ô#Ð#Ð#Ð#rB   ÚmeanÚtensorr    Ú	reduce_opc                 óT   — t          |t          ¦  «        rt          |||¬¦  «        S |S )a  Reduces a tensor from several distributed processes to one aggregated tensor.

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to gather results from. Defaults to all processes (world)
            reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
                Can also be a string 'sum' to calculate the sum during reduction.

        Return:
            reduced value, except when the input was not a tensor the output remains is unchanged

        )rk   )Ú
isinstancer   r   )r?   rj   r    rk   s       rA   Ú
all_reducezDDPStrategy.all_reduceˆ   s1   € õ  �f�fÑ%Ô%ð 	NÝ)¨&°%À9ÐMÑMÔMÐMØˆrB   Úargsc                 óÚ  — t          ¦   «         sd S t          j                             ¦   «         dk    r4t          j                             |                      ¦   «         ¬¦  «         d S 	 t          j                             ¦   «          d S # t          $ rX}dt          |¦  «        v r;t          j        d| j	        ¬¦  «        }t          j         
                    |¦  «         n‚ Y d }~d S d }~ww xY w)NÚnccl)r]   ÚPrivateUse1HooksInterfaceg        )Údevice)r   r`   ÚdistributedÚget_backendÚbarrierr_   ÚRuntimeErrorÚstrrj   rG   rn   )r?   ro   r2   ÚeÚdummy_tensors        rA   rv   zDDPStrategy.barrierœ   sõ   € å*Ñ,Ô,ð 	ØˆFÝÔ×(Ò(Ñ*Ô*¨fÒ4Ð4ÝÔ×%Ò%°×1OÒ1OÑ1QÔ1QÐ%ÑRÔRÐRÐRÐRð	ÝÔ!×)Ò)Ñ+Ô+Ð+Ð+Ð+øÝð ð ð Ø.µ#°a±&´&Ð8Ð8õ $)¤<°¸DÔ<LÐ#MÑ#MÔ#M�LÝÔ%×0Ò0°Ñ>Ô>Ð>Ð>àð ?Ð>Ð>Ð>Ð>Ð>øøøøðøøøs   Á(B Â
C*ÂAC%Ã%C*r   ÚobjÚsrcc                 ó�   — t          ¦   «         s|S |g}t          j                             ||t          j        ¬¦  «         |d         S )Nr   r   )r   r`   rt   Úbroadcast_object_listÚ_groupÚWORLD)r?   r{   r|   s      rA   Ú	broadcastzDDPStrategy.broadcast¯   sE   € å*Ñ,Ô,ð 	ØˆJàˆeˆÝÔ×/Ò/°°SÅÄÐ/ÑMÔMÐMØ�1ŒvˆrB   c                 ó~   •— t          |t          ¦  «        r|j        }t          ¦   «                              |¦  «        S rD   )rm   r   r[   r6   Úget_module_state_dict)r?   r[   r@   s     €rA   rƒ   z!DDPStrategy.get_module_state_dict¸   s5   ø€ å�fÕ5Ñ6Ô6ð 	#Ø”]ˆFÝ‰wŒw×,Ò,¨VÑ4Ô4Ð4rB   TÚ
state_dictÚstrictc                 óˆ   •— t          |t          ¦  «        r|j        }t          ¦   «                              |||¬¦  «         d S )N)r[   r„   r…   )rm   r   r[   r6   Úload_module_state_dict)r?   r[   r„   r…   r@   s       €rA   r‡   z"DDPStrategy.load_module_state_dict¾   sE   ø€ õ �fÕ5Ñ6Ô6ð 	#Ø”]ˆFÝ‰Œ×&Ò&¨fÀÐTZÐ&Ñ[Ô[Ð[Ð[Ð[rB   Ústrategy_registryc                 óˆ   — d}|D ]"\  }}|                      || d|›d�|¬¦  «         Œ#|                      d| ddd¬	¦  «         d S )
N))Úddpr&   )Ú	ddp_spawnr/   )r"   r0   )r#   r0   z DDP strategy with `start_method=ú`)Údescriptionr.   Úddp_find_unused_parameters_truezBAlias for `find_unused_parameters_true` and `start_method='popen'`Tr&   )r�   Úfind_unused_parametersr.   )Úregister)Úclsrˆ   ÚentriesÚnamer.   s        rA   Úregister_strategieszDDPStrategy.register_strategiesÆ   s�   € ð
ˆð #*ð 	ð 	ÑˆD�,Ø×&Ò&ØØØP¸|ÐPÐPÐPØ)ð	 'ñ ô ð ð ð 	×"Ò"Ø-ØØ\Ø#'Ø ð 	#ñ 	
ô 	
ð 	
ð 	
ð 	
rB   c                 óú   — |                       ¦   «          |                      ¦   «         | _        | j        €J ‚d| j        i}t
          r| j        j        dk    r| j        nd |d<   t          | j        | j        fi |¤Ž d S )Nr-   ÚcpuÚ	device_id)	Ú_set_world_ranksÚ_get_process_group_backendr9   r)   r:   r!   rG   Útyper   )r?   r2   s     rA   rZ   zDDPStrategy._setup_distributedÞ   s”   € Ø×ÒÑÔÐØ&*×&EÒ&EÑ&GÔ&GˆÔ#ØÔ'Ð3Ð3Ð3Ø"+¨T¬]Ð!;ˆÝ#ð 	_Ø6:Ô6FÔ6KÈuÒ6TÐ6T $Ô"2Ð"2ÐZ^ˆF�;ÑÝ˜dÔ6¸Ô8SÐ^Ð^ÐW]Ð^Ð^Ð^Ð^Ð^rB   c                 ó8   — | j         pt          | j        ¦  «        S rD   )r9   r   rG   rF   s    rA   r™   z&DDPStrategy._get_process_group_backendç   s   € ØÔ*ÐmÕ.[Ð\`Ô\lÑ.mÔ.mÐmrB   c                 óú   — | j         �V| j                              | j        | j        z  | j        z   ¦  «         | j                              | j        | j        z  ¦  «         | j        xt          _	        t          _	        d S rD   )r)   Úset_global_rankÚ	node_rankrN   rE   Úset_world_sizerJ   rR   r
   rQ   Úutils_rank_zero_onlyrF   s    rA   r˜   zDDPStrategy._set_world_ranksê   sr   € ØÔ#Ð/ØÔ$×4Ò4°T´^ÀdÔFXÑ5XÐ[_Ô[jÑ5jÑkÔkÐkØÔ$×3Ò3°D´NÀTÔEWÑ4WÑXÔXÐXð ;?Ô:JÐJ�ÔÕ2Ô7Ð7Ð7rB   c                 ó@   — | j         j        dk    rd n| j         j        gS )Nr–   )rG   rš   ÚindexrF   s    rA   r_   z%DDPStrategy._determine_ddp_device_idsò   s%   € ØÔ'Ô,°Ò5Ð5ˆtˆt¸DÔ<LÔ<RÐ;SÐSrB   )r3   N)Nri   )r   )T)4Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Úlistr`   rs   r   r   r   rx   r   r   r   r7   Úpropertyr   rG   ÚintrJ   ÚsetterrN   ÚdictrS   r,   rW   rY   r   r   re   rh   r   r	   r   rn   rv   r   r�   rƒ   Úboolr‡   Úclassmethodr   r”   rZ   r™   r˜   r_   Ú__classcell__)r@   s   @rA   r%   r%   5   s  ø€ € € € € ØUÐUð .2Ø9=Ø<@Ø04Ø)-Ø/3Ø'9ØHOð"ð "à˜kÔ*ð"ð # 4¨¬Ô#5Ô6ð"ð &Ð&8Ô9ð	"ð
   Ô-ð"ð ˜IÔ&ð"ð  (¨œ}ð"ð ˜)Ô$ð"ð ÐDÔEð"ð ð"ð 
ð"ð "ð "ð "ð "ð "ð4 Øð6˜Uœ\ð 6ð 6ð 6ñ „Xñ „Xð6ð ð˜3ð ð ð ñ „Xðð Ôð$ 3ð $¨4ð $ð $ð $ñ Ôð$ð ðV˜sð Vð Vð Vñ „XðVð Øða¨D°°c°¬Nð að að añ „Xñ „Xðað ð+ x°¤}ð +ð +ð +ñ „Xð+ð ð]ð ]ð ]ñ „Xð]ð ð"ð "ð "ð "ð "ñ „Xð"ð ðe 6ð eÐ.Eð eð eð eñ „Xðeð ð$ vð $°$ð $ð $ð $ñ „Xð$ð àgmðð ØðØ%-¨c¤]ðØFNÈuÐU]Ð_bÐUbÔOcÔFdðà	ðð ð ñ „Xðð& ð˜Sð ¨Cð °Dð ð ð ñ „Xðð$ ðð ˜Zð ¨cð ¸*ð ð ð ñ „Xðð ð5¨Fð 5°t¸CÀÀsÈFÀ{ÔASÐ<SÔ7Tð 5ð 5ð 5ð 5ð 5ñ „Xð5ð
 àX\ð\ð \Øð\Ø*.¨s°E¸#¸v¸+Ô4FÐ/FÔ*Gð\ØQUð\à	ð\ð \ð \ð \ð \ñ „Xð\ð Øð
Ð4Eð 
È$ð 
ð 
ð 
ñ „Xñ „[ð
ð,_ð _ð _ð _ðn¨Cð nð nð nð nðKð Kð Kð KðT¨8°D¸´IÔ+>ð Tð Tð Tð Tð Tð Tð Tð TrB   r%   c                   ó2   — e Zd Zedededefd„¦   «         ZdS )r<   r[   Úenabledr3   c                 óÈ   — |st          ¦   «         S t          |t          ¦  «        s*t          d| j        j        › d|j        j        › d�¦  «        ‚|                     ¦   «         S )z{Blocks gradient synchronization inside the :class:`~torch.nn.parallel.distributed.DistributedDataParallel`
        wrapper.zABlocking backward sync is only possible if the module passed to `zA.no_backward_sync` is wrapped in `DistributedDataParallel`. Got: ú.)r   rm   r   Ú	TypeErrorr@   r£   Úno_sync)r?   r[   r°   s      rA   Úno_backward_syncz(_DDPBackwardSyncControl.no_backward_sync÷   s|   € ð ð 	!Ý‘=”=Ð å˜&Õ"9Ñ:Ô:ð 	Ýð6Ø”^Ô,ð6ð 6àÔ)Ô2ð6ð 6ð 6ñô ð ð
 �~Š~ÑÔÐrB   N)r£   r¤   r¥   r   r   r¬   r   rµ   r^   rB   rA   r<   r<   ö   sH   € € € € € Øð  vð  ¸ð  ÐAWð  ð  ð  ñ „Xð ð  ð  rB   r<   )9Ú
contextlibr   r   Údatetimer   Útypingr   r   r   r	   r`   Útorch.distributedÚ"lightning_utilities.core.rank_zeror
   r    r   Útorch.nnr   Útorch.nn.parallel.distributedr   Útyping_extensionsr   Ú)lightning_fabric.accelerators.acceleratorr   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Ú9lightning_fabric.plugins.environments.cluster_environmentr   Ú)lightning_fabric.plugins.io.checkpoint_ior   Ú"lightning_fabric.plugins.precisionr   Ú5lightning_fabric.strategies.launchers.multiprocessingr   Ú7lightning_fabric.strategies.launchers.subprocess_scriptr   Ú$lightning_fabric.strategies.parallelr   Ú$lightning_fabric.strategies.registryr   Ú$lightning_fabric.strategies.strategyr   r   Ú&lightning_fabric.utilities.distributedr   r   r   r   r   r    r   Ú"lightning_fabric.utilities.importsr!   Ú$lightning_fabric.utilities.rank_zeroÚ_DDP_FORK_ALIASESr%   r<   r^   rB   rA   ú<module>rÌ      sv  ðð ;Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ø Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à €€€Ø Ð Ð Ð Ø UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AØ &Ð &Ð &Ð &Ð &Ð &à AÐ AÐ AÐ AÐ AÐ AØ TÐ TÐ TÐ TÐ TÐ TØ XÐ XÐ XÐ XÐ XÐ XØ BÐ BÐ BÐ BÐ BÐ BØ 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø ZÐ ZÐ ZÐ ZÐ ZÐ ZØ ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø AÐ AÐ AÐ AÐ AÐ AØ BÐ BÐ BÐ BÐ BÐ BØ QÐ QÐ QÐ QÐ QÐ QÐ QÐ Qðð ð ð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ GÐ GÐ GÐ GÐ GÐ GØ ?Ð ?Ð ?Ð ?Ð ?Ð ?ðÐ ð~Tð ~Tð ~Tð ~Tð ~TÐ"ñ ~Tô ~Tð ~TðB ð  ð  ð  ð  Ð2ñ  ô  ð  ð  ð  rB   