§
    “©ºjEN  ã                   ó  — d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlZd dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z' d dl#m(Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6m7Z7m8Z8 d dl9m:Z: d dl;m<Z<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZD d dlEmFZF d dlGmHZHmIZImZ erd dlJmKZK  e jL        eM¦  «        ZNdZO G d„ d e?¦  «        ZP G d!„ d"eB¦  «        ZQdS )#é    N)Únullcontext)Ú	timedelta)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚUnion)Úrank_zero_only)ÚTensor)ÚModule)ÚDistributedDataParallel)Ú	Optimizer)Úoverride)ÚCheckpointIOÚClusterEnvironment)Údefault_pg_timeout)Ú_StrategyRegistry)Ú_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_IS_WINDOWSÚ_TORCH_GREATER_EQUAL_2_3)Ú_optimizers_to_device)Ú
reset_seed)ÚReduceOp)ÚLightningOptimizer)Ú_register_ddp_comm_hookÚ_sync_module_statesÚprepare_for_backward)Ú	Precision)Ú_MultiProcessingLauncherÚ_SubprocessScriptLauncher)ÚParallelStrategy)Ú
TBroadcastÚ_ForwardRedirection)Ú	TrainerFn©Ú_augment_message)Úrank_zero_deprecationÚrank_zero_infor   )ÚModelAverager)Úddp_forkÚ%ddp_fork_find_unused_parameters_falseÚ$ddp_fork_find_unused_parameters_trueÚddp_notebookÚ)ddp_notebook_find_unused_parameters_falseÚ(ddp_notebook_find_unused_parameters_truec                   óº  ‡ — e Zd ZdZddddddddddedfded         deeej                          dee	         dee
         d	ee         d
ee         dee         dee         dee         dee         dee         ded         deddfˆ fd„Zedefd„¦   «         Zeedej        fd„¦   «         ¦   «         Zedefd„¦   «         Zej        deddfd„¦   «         Zedefd„¦   «         Zeedeeef         fd„¦   «         ¦   «         Zedee         fd„¦   «         ZedCd„¦   «         ZedCˆ fd„¦   «         Z edDd!„¦   «         Z!ed"e"de#fd#„¦   «         Z$dCd$„Z%defd%„Z&dCd&„Z'dCd'„Z(dCd(„Z)e	 dEd)e*d*eg ef         d"ee+d+e"f                  dedef
ˆ fd,„¦   «         Z,dCd-„Z-deee                  fd.„Z.ed/ededdfd0„¦   «         Z/edFd2e0d3ede0fd4„¦   «         Z1ed5e2ddfd6„¦   «         Z3ed5e2ddfd7„¦   «         Z4edCd8„¦   «         Z5e	 dGd:e2d;ee         d<ee+e6ef                  de2fd=„¦   «         Z7e8ed>e9ddfd?„¦   «         ¦   «         Z:ed@e;ddfdA„¦   «         Z<edCˆ fdB„¦   «         Z=ˆ xZ>S )HÚDDPStrategyzKStrategy for multi-process single-device training on one or multiple nodes.NÚpopenÚacceleratorzpl.accelerators.AcceleratorÚparallel_devicesÚcluster_environmentÚcheckpoint_ioÚprecision_pluginÚddp_comm_stateÚddp_comm_hookÚddp_comm_wrapperÚmodel_averaging_periodÚprocess_group_backendÚtimeoutÚstart_method)r8   ÚspawnÚforkÚ
forkserverÚkwargsÚreturnc                 ób  •— t          ¦   «                              |||||¬¦  «         t                               | j        j        › d�¦  «         t          ¦   «         | _        d| _        || _	        || _
        || _        || _        |	| _        d | _        |
| _        || _        || _        d| _        d S )N)r9   r:   r;   r<   r=   z: initializing DDP strategyé   F)ÚsuperÚ__init__ÚlogÚdebugÚ	__class__Ú__name__Ú_DDPForwardRedirectionÚ_forward_redirectionÚ
_num_nodesÚ_ddp_kwargsÚ_ddp_comm_stateÚ_ddp_comm_hookÚ_ddp_comm_wrapperÚ_model_averaging_periodÚ_model_averagerÚ_process_group_backendÚ_timeoutÚ_start_methodÚ_pl_static_graph_delay_done)Úselfr9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rH   rP   s                 €úp/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/strategies/ddp.pyrM   zDDPStrategy.__init__G   sÁ   ø€ õ  	‰Œ×ÒØ#Ø-Ø 3Ø'Ø-ð 	ñ 	
ô 	
ð 	
õ 	�	Š	�T”^Ô,ÐIÐIÐIÑJÔJÐJÝ$:Ñ$<Ô$<ˆÔ!ØˆŒØ!ˆÔØ-ˆÔØ+ˆÔØ!1ˆÔØ'=ˆÔ$Ø8<ˆÔØ5JˆÔ#Ø-4ˆŒØ)ˆÔØ+0ˆÔ(Ð(Ð(ó    c                 óT   — t          dt          | ¦  «        j        › d�d¬¦  «         dS )z1Legacy property kept for backwards compatibility.ú`z3.is_distributed` is deprecated. Use is discouraged.é   )Ú
stacklevelT)r-   ÚtyperQ   ©r_   s    r`   Úis_distributedzDDPStrategy.is_distributedl   s<   € õ 	ØX•�T‘
”
Ô#ÐXÐXÐXÐefð	
ñ 	
ô 	
ð 	
ð ˆtra   c                 ó8   — | j         €J ‚| j         | j                 S ©N)r:   Ú
local_rankrg   s    r`   Úroot_devicezDDPStrategy.root_devicet   s#   € ð Ô$Ð0Ð0Ð0ØÔ$ T¤_Ô5Ð5ra   c                 ó   — | j         S rj   ©rT   rg   s    r`   Ú	num_nodeszDDPStrategy.num_nodesz   s
   € àŒÐra   ro   c                 ó   — || _         d S rj   rn   )r_   ro   s     r`   ro   zDDPStrategy.num_nodes~   s   € ð $ˆŒˆˆra   c                 ó<   — | j         �t          | j         ¦  «        ndS ©Nr   )r:   Úlenrg   s    r`   Únum_processeszDDPStrategy.num_processesƒ   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUra   c                 ó0   — | j         | j        z  | j        dœS )N)Únum_replicasÚrank)ro   rt   Úglobal_rankrg   s    r`   Údistributed_sampler_kwargsz&DDPStrategy.distributed_sampler_kwargs‡   s   € ð "&¤°$Ô2DÑ!DÈtÔO_Ð`Ð`Ð`ra   c                 ó   — | j         S rj   )r[   rg   s    r`   rB   z!DDPStrategy.process_group_backendŒ   s   € àÔ*Ð*ra   c                 ó²   — | j         €J ‚| j        dk    r't          | j         | j        | j        ¦  «        | _        d S t          | | j        ¬¦  «        | _        d S )Nr8   )rD   )r;   r]   r&   rt   ro   Ú	_launcherr%   rg   s    r`   Ú_configure_launcherzDDPStrategy._configure_launcher�   s\   € àÔ'Ð3Ð3Ð3ØÔ Ò(Ð(Ý6°tÔ7OÐQUÔQcÐeiÔesÑtÔtˆDŒNˆNˆNå5°dÈÔI[Ð\Ñ\Ô\ˆDŒNˆNˆNra   c                 óp   •— t          ¦   «                              ¦   «          |                      ¦   «          d S rj   )rL   Úsetup_environmentÚsetup_distributed)r_   rP   s    €r`   r   zDDPStrategy.setup_environment˜   s1   ø€ å‰Œ×!Ò!Ñ#Ô#Ð#Ø×ÒÑ Ô Ð Ð Ð ra   Útrainerú
pl.Trainerc                 óì  — | j         €J ‚| j                              |¦  «         |j        j        }| j        €J ‚|t
          j        k    r+| j        r$| j                             | j        ¦  «        | _        | j	         
                    | j        ¦  «         |                      ¦   «          |t
          j        k    r*|                      ¦   «          |                      |¦  «         nt          | j        ¦  «         |                      ¦   «          |t
          j        k    rYt!          | j        | j        ¦  «         dd lmc mc mc m} t1          | j        |j        ¦  «        r|                      ¦   «          d S d S d S rr   )r9   ÚsetupÚstateÚfnÚmodelr*   ÚFITTINGÚ_layer_syncÚapplyr=   Úconvert_moduleÚmodel_to_deviceÚconfigure_ddpÚsetup_optimizersr"   Úsetup_precision_pluginr   Ú
optimizersrl   Ú>torch.distributed.algorithms.ddp_comm_hooks.post_localSGD_hookÚdistributedÚ
algorithmsÚddp_comm_hooksÚpost_localSGD_hookÚ
isinstancerV   ÚPostLocalSGDStateÚ_enable_model_averaging)r_   r�   Ú
trainer_fnÚpost_localSGDs       r`   r„   zDDPStrategy.setup�   sŠ  € àÔÐ+Ð+Ð+ØÔ×Ò˜wÑ'Ô'Ð'à”]Ô%ˆ
ØŒzÐ%Ð%Ð%Ø�Ô*Ò*Ð*¨tÔ/?Ð*ØÔ)×/Ò/°´
Ñ;Ô;ˆDŒJàÔ×,Ò,¨T¬ZÑ8Ô8Ð8Ø×ÒÑÔÐà�Ô*Ò*Ð*à×ÒÑ Ô Ð ð ×!Ò! 'Ñ*Ô*Ð*Ð*õ   ¤
Ñ+Ô+Ð+Ø×#Ò#Ñ%Ô%Ð%Ø�Ô*Ò*Ð*Ý! $¤/°4Ô3CÑDÔDÐDàbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbÐbå˜$Ô.°Ô0OÑPÔPð /Ø×,Ò,Ñ.Ô.Ð.Ð.Ð.ð +Ð*ð
/ð /ra   r‡   c                 ój  — |                       ¦   «         }t                               d|› d| j        › �¦  «         |�;t          j                             t          j                             ¦   «         ¦  «        nt          ¦   «         }|5  t          d||dœ| j        ¤Žcddd¦  «         S # 1 swxY w Y   dS )z^Wraps the model into a :class:`~torch.nn.parallel.distributed.DistributedDataParallel` module.z&setting up DDP model with device ids: z
, kwargs: N)ÚmoduleÚ
device_ids© )
Údetermine_ddp_device_idsrN   rO   rU   ÚtorchÚcudaÚstreamÚStreamr   r   )r_   r‡   r�   Úctxs       r`   Ú_setup_modelzDDPStrategy._setup_model¼   s
  € ð ×2Ò2Ñ4Ô4ˆ
Ý�	Š	Ðc¸:ÐcÐcÐQUÔQaÐcÐcÑdÔdÐdà8BÐ8N�eŒj×Ò¥¤
× 1Ò 1Ñ 3Ô 3Ñ4Ô4Ð4ÕT_ÑTaÔTaˆØð 	dð 	dÝ*Ðc°%ÀJÐcÐcÐRVÔRbÐcÐcð	dð 	dð 	dð 	dñ 	dô 	dð 	dð 	dð 	dð 	dð 	dð 	døøøð 	dð 	dð 	dð 	dð 	dð 	ds   ÂB(Â(B,Â/B,c                 ód  — t                                | j        j        › d�¦  «         t	          ¦   «          |                      ¦   «          |                      ¦   «         | _        | j        €J ‚d| j	        i}t          r| j        j        dk    r| j        nd |d<   t          | j        | j        fi |¤Ž d S )Nz: setting up distributed...rC   ÚcpuÚ	device_id)rN   rO   rP   rQ   r   Úset_world_ranksÚ_get_process_group_backendr[   r;   r\   r   rl   rf   r   )r_   rH   s     r`   r€   zDDPStrategy.setup_distributedÆ   s½   € Ý�	Š	�T”^Ô,ÐIÐIÐIÑJÔJÐJÝ‰ŒˆØ×ÒÑÔÐØ&*×&EÒ&EÑ&GÔ&GˆÔ#ØÔ'Ð3Ð3Ð3Ø"+¨T¬]Ð!;ˆÝ#ð 	_Ø6:Ô6FÔ6KÈuÒ6TÐ6T $Ô"2Ð"2ÐZ^ˆF�;ÑÝ˜dÔ6¸Ô8SÐ^Ð^ÐW]Ð^Ð^Ð^Ð^Ð^ra   c                 ó8   — | j         pt          | j        ¦  «        S rj   )r[   r   rl   rg   s    r`   rª   z&DDPStrategy._get_process_group_backendÑ   s   € ØÔ*ÐmÕ.[Ð\`Ô\lÑ.mÔ.mÐmra   c                 óú   — | j         �V| j                              | j        | j        z  | j        z   ¦  «         | j                              | j        | j        z  ¦  «         | j        xt          _	        t          _	        d S rj   )r;   Úset_global_rankÚ	node_rankrt   rk   Úset_world_sizero   rx   r   rw   Úutils_rank_zero_onlyrg   s    r`   r©   zDDPStrategy.set_world_ranksÔ   sr   € ØÔ#Ð/ØÔ$×4Ò4°T´^ÀdÔFXÑ5XÐ[_Ô[jÑ5jÑkÔkÐkØÔ$×3Ò3°D´NÀTÔEWÑ4WÑXÔXÐXð ;?Ô:JÐJ�ÔÕ2Ô7Ð7Ð7ra   c                 óþ   — t                                | j        j        › d�¦  «         | j        j        dk    rEt          | j        t          ¦  «        sJ ‚t          | j        | j
        | j        | j        ¬¦  «         d S d S )Nz: registering ddp hooksr¡   )r‡   r>   r?   r@   )rN   rO   rP   rQ   rl   rf   r–   r‡   r   r!   rV   rW   rX   rg   s    r`   Ú_register_ddp_hookszDDPStrategy._register_ddp_hooksÜ   sŽ   € Ý�	Š	�T”^Ô,ÐEÐEÐEÑFÔFÐFð ÔÔ  FÒ*Ð*Ý˜dœjÕ*AÑBÔBÐBÐBÐBÝ#Ø”jØ#Ô3Ø"Ô1Ø!%Ô!7ð	ñ ô ð ð ð ð +Ð*ra   c                 ó  — t                                | j        j        › d�¦  «         | j        €t          d¦  «        ‚ddlm}m}m	} | j
        D ]h}t          |t          ¦  «        r|j        }t          st          ||¦  «        nd}t          |||f¦  «        s|rt          d|j        j        › d�¦  «        ‚Œi| j        €J ‚t           j        j        j        j                             | j        | j        j        ¬¦  «        | _        d S )	Nz.: reinitializing optimizers with post localSGDz\Post-localSGD algorithm is used, but model averaging period is not provided to DDP strategy.r   )ÚDistributedOptimizerÚPostLocalSGDOptimizerÚZeroRedundancyOptimizerFzKCurrently model averaging cannot work with a distributed optimizer of type ú.)ÚperiodÚwarmup_steps)rN   rO   rP   rQ   rY   Ú
ValueErrorÚtorch.distributed.optimr´   rµ   r¶   r�   r–   r    Ú
_optimizerr   rV   r    r’   r“   Úmodel_averagingÚ	averagersÚPeriodicModelAveragerÚstart_localSGD_iterrZ   )r_   r´   rµ   r¶   Ú	optimizerÚis_distributed_optimizers         r`   r˜   z#DDPStrategy._enable_model_averagingé   sM  € Ý�	Š	�T”^Ô,Ð\Ð\Ð\Ñ]Ô]Ð]ØÔ'Ð/ÝØnñô ð ð 	qÐpÐpÐpÐpÐpÐpÐpÐpÐpàœð 		ð 		ˆIÝ˜)Õ%7Ñ8Ô8ð 1Ø%Ô0�	åZeÐ'p¥z°)Ð=QÑ'RÔ'RÐ'RÐkpÐ$Ý˜)Ð&=Ð?TÐ%UÑVÔVð ÐZrð Ý ð7Ø Ô*Ô3ð7ð 7ð 7ñô ð ðð Ô#Ð/Ð/Ð/Ý$Ô0Ô;ÔKÔU×kÒkØÔ/¸dÔ>RÔ>fð  lñ  
ô  
ˆÔÐÐra   rÁ   Úclosureúpl.LightningModulec                 óÂ   •—  t          ¦   «         j        |||fi |¤Ž}| j        €|S d„ |j        D ¦   «         }| j                             t          |¦  «        ¦  «         |S )aI  Performs the actual optimizer step.

        Args:
            optimizer: the optimizer performing the step
            closure: closure calculating the loss value
            model: reference to the model, optionally defining optimizer step related hooks
            **kwargs: Any extra arguments to ``optimizer.step``

        Nc                 ó6   — g | ]}|d          D ]}|j         ®	|‘ŒŒS )Úparams)Úgrad)Ú.0r   Úparams      r`   ú
<listcomp>z.DDPStrategy.optimizer_step.<locals>.<listcomp>  s2   € ÐsÐsÐs˜EÈÈxÌÐsÐsÀÐ\aÔ\fÐ\r�%Ð\rÐ\rÐ\rÐ\rra   )rL   Úoptimizer_steprZ   Úparam_groupsÚaverage_parametersÚiter)r_   rÁ   rÃ   r‡   rH   Úoptimizer_outputrÇ   rP   s          €r`   rÌ   zDDPStrategy.optimizer_step  sp   ø€ ð" 2�5™7œ7Ô1°)¸WÀeÐVÐVÈvÐVÐVÐàÔÐ'Ø#Ð#àsÐs YÔ%;ÐsÑsÔsˆØÔ×/Ò/µ°V±´Ñ=Ô=Ð=àÐra   c                 óü   — t                                | j        j        › d�¦  «         t	          | j        t          j        ¦  «        sJ ‚|                      | j        ¦  «        | _        |  	                    ¦   «          d S )Nz%: configuring DistributedDataParallel)
rN   rO   rP   rQ   r–   r‡   ÚplÚLightningModuler¥   r²   rg   s    r`   r�   zDDPStrategy.configure_ddp  sl   € Ý�	Š	�T”^Ô,ÐSÐSÐSÑTÔTÐTÝ˜$œ*¥bÔ&8Ñ9Ô9Ð9Ð9Ð9Ø×&Ò& t¤zÑ2Ô2ˆŒ
Ø× Ò Ñ"Ô"Ð"Ð"Ð"ra   c                 ó@   — | j         j        dk    rd S | j         j        gS )Nr§   )rl   rf   Úindexrg   s    r`   rŸ   z$DDPStrategy.determine_ddp_device_ids"  s'   € ØÔÔ  EÒ)Ð)Ø�4ØÔ Ô&Ð'Ð'ra   Úargsc                 ó  — t          ¦   «         sd S t          j                             ¦   «         dk    r4t          j                             |                      ¦   «         ¬¦  «         d S t          j                             ¦   «          d S )NÚnccl)r�   )r   r    r’   Úget_backendÚbarrierrŸ   )r_   rÖ   rH   s      r`   rÚ   zDDPStrategy.barrier'  sv   € å*Ñ,Ô,ð 	ØˆFåÔ×(Ò(Ñ*Ô*¨fÒ4Ð4ÝÔ×%Ò%°×1NÒ1NÑ1PÔ1PÐ%ÑQÔQÐQÐQÐQåÔ×%Ò%Ñ'Ô'Ð'Ð'Ð'ra   r   ÚobjÚsrcc                 ó�   — t          ¦   «         s|S |g}t          j                             ||t          j        ¬¦  «         |d         S )Nr   r   )r   r    r’   Úbroadcast_object_listÚ_groupÚWORLD)r_   rÛ   rÜ   s      r`   Ú	broadcastzDDPStrategy.broadcast1  sE   € å*Ñ,Ô,ð 	ØˆJàˆeˆÝÔ×/Ò/°°SÅÄÐ/ÑMÔMÐMØ�1Œvˆra   Úclosure_lossc                 ó–   — t          | j        t          ¦  «        sdS | j        €J ‚| j        j        st          | j        |¦  «         dS dS )z.Run before precision plugin executes backward.N)r–   r‡   r   Úlightning_moduleÚautomatic_optimizationr#   )r_   râ   s     r`   Úpre_backwardzDDPStrategy.pre_backward:  s]   € õ ˜$œ*Õ&=Ñ>Ô>ð 	ØˆFØÔ$Ð0Ð0Ð0ØÔ$Ô;ð 	;Ý  ¤¨\Ñ:Ô:Ð:Ð:Ð:ð	;ð 	;ra   c                 óâ   — | j         }| j        }t          |t          ¦  «        sd S |�|j        rd S t          |dd¦  «        sd S | j        rd S |j        }|                     ¦   «          d| _        d S )NÚstatic_graphFT)	r‡   rä   r–   r   rå   Úgetattrr^   ÚreducerÚ_delay_all_reduce)r_   râ   r‡   Úlmrê   s        r`   Úpost_backwardzDDPStrategy.post_backwardC  sŽ   € ð ”
ˆØÔ"ˆÝ˜%Õ!8Ñ9Ô9ð 	ØˆFØˆ:˜Ô2ˆ:ØˆFÝ�u˜n¨eÑ4Ô4ð 	ØˆFØÔ+ð 	ØˆFð ”-ˆØ×!Ò!Ñ#Ô#Ð#à+/ˆÔ(Ð(Ð(ra   c                 ó´   — t                                | j        j        › d| j        › d�¦  «         | j        €J ‚| j                             | j        ¦  «         d S )Nz: moving model to device [z]...)rN   rO   rP   rQ   rl   r‡   Útorg   s    r`   rŒ   zDDPStrategy.model_to_deviceX  sX   € å�	Š	�T”^Ô,Ð^Ð^ÈÔHXÐ^Ð^Ð^Ñ_Ô_Ð_ØŒzÐ%Ð%Ð%ØŒ
�Š�dÔ&Ñ'Ô'Ð'Ð'Ð'ra   ÚmeanÚtensorr   Ú	reduce_opc                 óT   — t          |t          ¦  «        rt          |||¬¦  «        S |S )a  Reduces a tensor from several distributed processes to one aggregated tensor.

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to gather results from. Defaults to all processes (world)
            reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
                Can also be a string 'sum' to calculate the sum during reduction.

        Return:
            reduced value, except when the input was not a tensor the output remains is unchanged

        )rò   )r–   r   r   )r_   rñ   r   rò   s       r`   ÚreducezDDPStrategy.reduce^  s1   € õ  �f�fÑ%Ô%ð 	NÝ)¨&°%À9ÐMÑMÔMÐMØˆra   Ústrategy_registryc           
      ó¬   — d}|D ]"\  }}|                      || d|› d�|¬¦  «         Œ#d}|D ]'\  }}}|                      || d|› d|› d�||¬¦  «         Œ(d S )	N))Úddpr8   )Ú	ddp_spawnrE   )r0   rF   )r3   rF   z"DDP strategy with `start_method` 'ú')ÚdescriptionrD   ))Ú ddp_find_unused_parameters_falseFr8   )Úddp_find_unused_parameters_trueTr8   )Ú&ddp_spawn_find_unused_parameters_falseFrE   )Ú%ddp_spawn_find_unused_parameters_trueTrE   )r1   FrF   )r2   TrF   )r4   FrF   )r5   TrF   z.DDP strategy with `find_unused_parameters` as z and `start_method` ')rú   Úfind_unused_parametersrD   )Úregister)Úclsrõ   ÚentriesÚnamerD   Úfups         r`   Úregister_strategieszDDPStrategy.register_strategiesr  sÆ   € ð
ˆð #*ð 	ð 	ÑˆD�,Ø×&Ò&ØØØPÀÐPÐPÐPØ)ð	 'ñ ô ð ð ð	
ˆð (/ð 	ð 	Ñ#ˆD�#�|Ø×&Ò&ØØØvÈSÐvÐvÐgsÐvÐvÐvØ'*Ø)ð 'ñ ô ð ð ð	ð 	ra   Ú	exceptionc                 ó*   — t          |dd¬¦  «         d S )Nz>.*Expected to have finished reduction in the prior iteration.*ay  It looks like your LightningModule has parameters that were not used in producing the loss returned by training_step. If this is intentional, you must enable the detection of unused parameters in DDP, either by setting the string value `strategy='ddp_find_unused_parameters_true'` or by setting the flag in the strategy with `strategy=DDPStrategy(find_unused_parameters=True)`.)ÚpatternÚnew_messager+   )r_   r  s     r`   Úon_exceptionzDDPStrategy.on_exception–  s/   € åØØTðtð			
ñ 		
ô 		
ð 		
ð 		
ð 		
ra   c                 óH  •— t                                | j        j        › d�¦  «         | j        }t          | j        t          ¦  «        r\| j        j        sI| j         	                    ¦   «          
                    d¦  «        rt          d| j        j        › d�¦  «         || _        |�Z|j        �S|j        j        j        t          j        k    r4| j        r-| j        €J ‚| j                             | j        ¦  «        | _        t'          ¦   «                              ¦   «          d S )Nz: tearing down strategyÚcan_set_static_graphzyYour model can run with static graph optimizations. For future training runs, we suggest you pass `Trainer(..., strategy=z%(static_graph=True))` to enable them.)rN   rO   rP   rQ   rä   r–   r‡   r   rè   Ú_get_ddp_logging_dataÚgetr.   Ú_trainerr…   r†   r*   rˆ   r‰   ÚrevertrL   Úteardown)r_   Ú	pl_modulerP   s     €r`   r  zDDPStrategy.teardown£  s(  ø€ å�	Š	�T”^Ô,ÐEÐEÐEÑFÔFÐFàÔ)ˆ	Ý�d”jÕ"9Ñ:Ô:ð 	#Ø”:Ô*ð ¨t¬z×/OÒ/OÑ/QÔ/Q×/UÒ/UÐVlÑ/mÔ/mð ÝðsØ48´NÔ4Kðsð sð sñô ð ð
 #ˆDŒJð Ð!ð Ô"Ð.ØÔ"Ô(Ô+­yÔ/@Ò@Ð@ØÔ ð Að ”:Ð)Ð)Ð)ØÔ)×0Ò0°´Ñ<Ô<ˆDŒJå‰Œ×ÒÑÔÐÐÐra   )rI   N)r�   r‚   rI   Nrj   )r   )Nrð   )?rQ   Ú
__module__Ú__qualname__Ú__doc__r   r	   Úlistr    Údevicer   r   r$   Úobjectr   ÚintÚstrr   r   r   rM   ÚpropertyÚboolrh   r   rl   ro   Úsetterrt   Údictry   rB   r}   r   r„   r   r   r¥   r€   rª   r©   r²   r˜   r   r
   rÌ   r�   rŸ   rÚ   r(   rá   r   ræ   rí   rŒ   r   rô   Úclassmethodr   r  ÚBaseExceptionr
  r  Ú__classcell__)rP   s   @r`   r7   r7   D   s3  ø€ € € € € ØUÐUð @DØ9=Ø<@Ø04Ø04Ø+/Ø,0Ø/3Ø04Ø/3Ø'9ØHOð#1ð #1àÐ;Ô<ð#1ð # 4¨¬Ô#5Ô6ð#1ð &Ð&8Ô9ð	#1ð
   Ô-ð#1ð # 9Ô-ð#1ð ! Ô(ð#1ð   Ô)ð#1ð # 8Ô,ð#1ð !)¨¤ð#1ð  (¨œ}ð#1ð ˜)Ô$ð#1ð ÐDÔEð#1ð ð#1ð 
ð#1ð #1ð #1ð #1ð #1ð #1ðJ ð ð ð ð ñ „Xðð Øð6˜Uœ\ð 6ð 6ð 6ñ „Xñ „Xð6ð ð˜3ð ð ð ñ „Xðð Ôð$ 3ð $¨4ð $ð $ð $ñ Ôð$ð ðV˜sð Vð Vð Vñ „XðVð Øða¨D°°c°¬Nð að að añ „Xñ „Xðað ð+ x°¤}ð +ð +ð +ñ „Xð+ð ð]ð ]ð ]ñ „Xð]ð ð!ð !ð !ð !ð !ñ „Xð!ð ð/ð /ð /ñ „Xð/ð< ðd &ð dÐ-Dð dð dð dñ „Xðdð	_ð 	_ð 	_ð 	_ðn¨Cð nð nð nð nðKð Kð Kð Kðð ð ð ð
ð 
ð 
ð 
ð0 ð
 @Dð	 ð  àð ð ˜"˜c˜'Ô"ð ð ˜Ð2°FÐ:Ô;Ô<ð	 ð
 ð ð 
ð ð  ð  ð  ð  ñ „Xð ð4#ð #ð #ð #ð(¨(°4¸´9Ô*=ð (ð (ð (ð (ð
 ð(˜Sð (¨Cð (°Dð (ð (ð (ñ „Xð(ð ðð ˜Zð ¨cð ¸*ð ð ð ñ „Xðð ð;¨ð ;°Dð ;ð ;ð ;ñ „Xð;ð ð0¨&ð 0°Tð 0ð 0ð 0ñ „Xð0ð( ð(ð (ð (ñ „Xð(ð
 àgmðð ØðØ%-¨c¤]ðØFNÈuÐU]Ð_bÐUbÔOcÔFdðà	ðð ð ñ „Xðð& Øð Ð4Eð  È$ð  ð  ð  ñ „Xñ „[ð ðD ð

 mð 

¸ð 

ð 

ð 

ñ „Xð

ð ðð ð ð ð ñ „Xðð ð ð ð ra   r7   c                   óV   — e Zd Zededdddfd„¦   «         Zededdddfd„¦   «         ZdS )rR   Úwrapper_moduleÚoriginal_modulerÄ   rI   Nc                 óT   — t          |t          ¦  «        r|j        sd|_        d S d S d S )NF©r–   r   rå   Úrequire_backward_grad_sync©r_   r#  r$  s      r`   Úon_after_inner_forwardz-_DDPForwardRedirection.on_after_inner_forwardÀ  sC   € õ �nÕ&=Ñ>Ô>ð 	>ÀÔGmð 	>Ø8=ˆNÔ5Ð5Ð5ð	>ð 	>ð 	>ð 	>ra   c                 óT   — t          |t          ¦  «        r|j        sd|_        d S d S d S )NTr&  r(  s      r`   Úon_after_outer_forwardz-_DDPForwardRedirection.on_after_outer_forwardÇ  sA   € å�nÕ&=Ñ>Ô>ð 	=ÀÔGmð 	=Ø8<ˆNÔ5Ð5Ð5ð	=ð 	=ð 	=ð 	=ra   )rQ   r  r  r   r   r)  r+  rž   ra   r`   rR   rR   ¿  s€   € € € € € Øð>°Vð >ÐNbð >Ðgkð >ð >ð >ñ „Xð>ð ð=°Vð =ÐNbð =Ðgkð =ð =ð =ñ „Xð=ð =ð =ra   rR   )RÚloggingÚ
contextlibr   Údatetimer   Útypingr   r   r   r   r	   r
   r    Útorch.distributedÚ"lightning_utilities.core.rank_zeror   r°   r   Útorch.nnr   Útorch.nn.parallel.distributedr   Útorch.optim.optimizerr   Útyping_extensionsr   Úpytorch_lightningrÒ   Úlightning_fabric.pluginsr   r   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Úlightning_fabric.strategiesr   Ú&lightning_fabric.utilities.distributedr   r   r   r   r   rß   Ú"lightning_fabric.utilities.importsr   r   Ú$lightning_fabric.utilities.optimizerr   Úlightning_fabric.utilities.seedr   Ú lightning_fabric.utilities.typesr   Ú pytorch_lightning.core.optimizerr    Ú'pytorch_lightning.overrides.distributedr!   r"   r#   Ú#pytorch_lightning.plugins.precisionr$   Ú&pytorch_lightning.strategies.launchersr%   r&   Ú%pytorch_lightning.strategies.parallelr'   Ú%pytorch_lightning.strategies.strategyr(   r)   Ú pytorch_lightning.trainer.statesr*   Ú&pytorch_lightning.utilities.exceptionsr,   Ú%pytorch_lightning.utilities.rank_zeror-   r.   Ú6torch.distributed.algorithms.model_averaging.averagersr/   Ú	getLoggerrQ   rN   Ú_DDP_FORK_ALIASESr7   rR   rž   ra   r`   ú<module>rK     s0  ðð €€€Ø "Ð "Ð "Ð "Ð "Ð "Ø Ð Ð Ð Ð Ð Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Ià €€€Ø Ð Ð Ð Ø UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AØ +Ð +Ð +Ð +Ð +Ð +Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ TÐ TÐ TÐ TÐ TÐ TØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TØ FÐ FÐ FÐ FÐ FÐ FØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø fÐ fÐ fÐ fÐ fÐ fÐ fÐ fØ BÐ BÐ BÐ BÐ BÐ BØ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø CÐ CÐ CÐ CÐ CÐ CØ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gàð UØTÐTÐTÐTÐTÐTà€gÔ˜Ñ!Ô!€ðÐ ðxð xð xð xð xÐ"ñ xô xð xðv=ð =ð =ð =ð =Ð0ñ =ô =ð =ð =ð =ra   