§
    “©ºjæG  ã                   ó  — d dl Z d dlmZmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlZd dlmZ d dlmZm Z m!Z!m"Z" d dl#m$Z$m%Z%m&Z&m'Z' d dl#m(Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZA d dlBmCZC d dlDmZ erd dlEmFZF  G d„ de=¦  «        ZGdeHeIef         dejJ        jK        d eHeIef         fd!„ZLdS )"é    N)Ú	GeneratorÚMapping)ÚcontextmanagerÚnullcontext)Ú	timedelta)ÚPath)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalÚUnion)Úrank_zero_only)ÚTensor)Ú	Optimizer)Úoverride)Údefault_pg_timeout)Ú_distributed_checkpoint_saveÚ_is_sharded_checkpointÚ_load_checkpointÚ_setup_device_mesh)Ú_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_TORCH_GREATER_EQUAL_2_3Ú_TORCH_GREATER_EQUAL_2_4)Ú_materialize_distributed_module)Ú_METADATA_FILENAME)Ú_optimizers_to_device)Ú
reset_seed)Ú_PATHÚReduceOp)ÚLightningOptimizer)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú
TBroadcast)Ú	TrainerFn)Úis_overridden)Ú
DeviceMeshc                   óâ  ‡ — e Zd ZdZddddefdeed         ef         deed         ef         dede	e
         d	e	e         d
dfˆ fd„Zed;d„¦   «         Zeed
ej        fd„¦   «         ¦   «         Zed
efd„¦   «         Zeed
ee
ef         fd„¦   «         ¦   «         Zed
e	e
         fd„¦   «         Zeed
efd„¦   «         ¦   «         Zeed
efd„¦   «         ¦   «         Zed<d„¦   «         Zed<ˆ fd„¦   «         Zed=d„¦   «         Zed=ˆ fd„¦   «         Zed<d„¦   «         Zeed>de	e         d
e d         fd„¦   «         ¦   «         Z!ed>de	e
         d
dfd„¦   «         Z"ed?d!e#d"ed
e#fd#„¦   «         Z$e	 	 d@d%ee%ef         d&e	e         d'e	ee&e
f                  d
e%fd(„¦   «         Z'd
e(e         fd)„Z)ed<d*„¦   «         Z*ed
ee
ef         fd+„¦   «         Z+edAd,e,e
ef         d-ed
dfd.„¦   «         Z-ed/e.d
ee
ef         fd0„¦   «         Z/ed,e,e
ef         d
dfd1„¦   «         Z0e	 d>d,ee
ef         d2e1d3e	e         d
dfˆ fd4„¦   «         Z2ed>d5e1d6e	e         d
ee
ef         fd7„¦   «         Z3d<ˆ fd8„Z4d
e
fd9„Z5d<d:„Z6ˆ xZ7S )BÚModelParallelStrategya–  Enables user-defined parallelism applied to a model.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Currently supports up to 2D parallelism. Specifically, it supports the combination of
    Fully Sharded Data-Parallel 2 (FSDP2) with Tensor Parallelism (DTensor). These PyTorch APIs are currently still
    experimental in PyTorch (see https://pytorch.org/docs/stable/distributed.tensor.parallel.html).
    Requires PyTorch 2.4 or newer.

    Arguments:
        data_parallel_size: The number of devices within a data-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of nodes in the cluster.
        tensor_parallel_size: The number of devices within a tensor-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of GPUs in a single node.
        save_distributed_checkpoint: If ``True``, each rank saves its shard of weights and optimizer states to a file.
            The checkpoint is a folder with as many files as the world size.
            If ``False``, the full weights and optimizer states get assembled on rank 0 and saved to a single file.

    ÚautoTNÚdata_parallel_sizeÚtensor_parallel_sizeÚsave_distributed_checkpointÚprocess_group_backendÚtimeoutÚreturnc                 ó   •— t          ¦   «                              ¦   «          t          s$t          t	          | ¦  «        j        › d�¦  «        ‚|| _        || _        || _        || _	        || _
        d | _        d| _        d S )Nz  requires PyTorch 2.4 or higher.é   )ÚsuperÚ__init__r   ÚImportErrorÚtypeÚ__name__Ú_data_parallel_sizeÚ_tensor_parallel_sizeÚ_save_distributed_checkpointÚ_process_group_backendÚ_timeoutÚ_device_meshÚ	num_nodes)Úselfr/   r0   r1   r2   r3   Ú	__class__s         €ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/strategies/model_parallel.pyr8   zModelParallelStrategy.__init__Q   s~   ø€ õ 	‰Œ×ÒÑÔÐÝ'ð 	XÝ¥ d¡¤Ô!4ÐVÐVÐVÑWÔWÐWØ#5ˆÔ Ø%9ˆÔ"Ø,GˆÔ)Ø5JˆÔ#Ø-4ˆŒØ26ˆÔØˆŒˆˆó    r+   c                 ó<   — | j         €t          d¦  «        ‚| j         S )NzKAccessing the device mesh before processes have initialized is not allowed.)rA   ÚRuntimeError©rC   s    rE   Údevice_meshz!ModelParallelStrategy.device_meshd   s#   € àÔÐ$ÝÐlÑmÔmÐmØÔ Ð rF   c                 ó8   — | j         €J ‚| j         | j                 S ©N)Úparallel_devicesÚ
local_rankrI   s    rE   Úroot_devicez!ModelParallelStrategy.root_devicej   s#   € ð Ô$Ð0Ð0Ð0ØÔ$ T¤_Ô5Ð5rF   c                 ó<   — | j         �t          | j         ¦  «        ndS )Nr   )rM   ÚlenrI   s    rE   Únum_processesz#ModelParallelStrategy.num_processesp   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUrF   c                 ó€   — | j         €J ‚| j         d         }|                     ¦   «         |                     ¦   «         dœS )NÚdata_parallel)Únum_replicasÚrank)rJ   ÚsizeÚget_local_rank)rC   Údata_parallel_meshs     rE   Údistributed_sampler_kwargsz0ModelParallelStrategy.distributed_sampler_kwargst   sH   € ð ÔÐ+Ð+Ð+Ø!Ô-¨oÔ>ÐØ 2× 7Ò 7Ñ 9Ô 9ÐCU×CdÒCdÑCfÔCfÐgÐgÐgrF   c                 ó   — | j         S rL   )r?   rI   s    rE   r2   z+ModelParallelStrategy.process_group_backend{   s   € àÔ*Ð*rF   c                 ó   — dS )NT© rI   s    rE   Úrestore_checkpoint_after_setupz4ModelParallelStrategy.restore_checkpoint_after_setup   s	   € ð ˆtrF   c                 ó   — dS )NFr]   rI   s    rE   Úlightning_restore_optimizerz1ModelParallelStrategy.lightning_restore_optimizer„   s	   € ð ˆurF   c                 ó~   — | j         €J ‚| j         j        s't          | j         | j        | j        ¦  «        | _        d S d S rL   )Úcluster_environmentÚcreates_processes_externallyr&   rR   rB   Ú	_launcherrI   s    rE   Ú_configure_launcherz)ModelParallelStrategy._configure_launcher‰   sN   € àÔ'Ð3Ð3Ð3ØÔ'ÔDð 	uÝ6°tÔ7OÐQUÔQcÐeiÔesÑtÔtˆDŒNˆNˆNð	uð 	urF   c                 óV  •— t          ¦   «                              ¦   «          |                      ¦   «          | j        dk    r| j        | _        | j        dk    r| j        | _        t          | j        | j        | j        | j	        ¦  «        | _
        | j        €J ‚| j
        | j        _
        d S )Nr.   )r7   Úsetup_environmentÚ_setup_distributedr<   rB   r=   rR   r   Ú
world_sizerO   rA   Úlightning_module)rC   rD   s    €rE   rg   z'ModelParallelStrategy.setup_environment�   s¨   ø€ å‰Œ×!Ò!Ñ#Ô#Ð#Ø×ÒÑ!Ô!Ð!ØÔ# vÒ-Ð-Ø'+¤~ˆDÔ$ØÔ%¨Ò/Ð/Ø)-Ô);ˆDÔ&Ý.ØÔ$ dÔ&@À$Ä/ÐSWÔScñ
ô 
ˆÔð Ô$Ð0Ð0Ð0Ø-1Ô->ˆÔÔ*Ð*Ð*rF   Útrainerú
pl.Trainerc                 ó<  ‡— ddl mŠ | j        €J ‚| j        €J ‚| j                             |¦  «         t          d| j        ¦  «        s%t          dt          | ¦  «        j	        › d�¦  «        ‚t          ˆfd„| j                             ¦   «         D ¦   «         ¦  «        rt          d| j        j	        › d�¦  «        ‚t          | j        | j        ¦  «         | j                             | j        ¦  «        | _        |                      ¦   «          |                      ¦   «          |j        j        t*          j        k    r|                      |¦  «         |                      ¦   «          |j        j        t*          j        k    rt3          | j        | j        ¦  «         d S d S )	Nr   ©ÚFullyShardedDataParallelÚconfigure_modelzWhen using the zs, you are required to override the `configure_model()` hook in the LightningModule and apply parallelization there.c              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S rL   )Ú
isinstance)Ú.0Úmodro   s     €rE   ú	<genexpr>z.ModelParallelStrategy.setup.<locals>.<genexpr>«   s.   øè è € ÐYÐY¸S�z˜#Ð7Ñ8Ô8ÐYÐYÐYÐYÐYÐYrF   z\Found modules that are wrapped with `torch.distributed.fsdp.FullyShardedDataParallel`. The `z5` only supports the new FSDP2 APIs in PyTorch >= 2.4.)Útorch.distributed.fsdpro   ÚmodelÚacceleratorÚsetupr*   rj   Ú	TypeErrorr:   r;   ÚanyÚmodulesrD   r   rO   Úprecision_pluginÚconvert_moduleÚmodel_to_deviceÚbarrierÚstateÚfnr)   ÚFITTINGÚsetup_optimizersÚsetup_precision_pluginr!   Ú
optimizers)rC   rk   ro   s     @rE   ry   zModelParallelStrategy.setupž   sº  ø€ àCÐCÐCÐCÐCÐCàŒzÐ%Ð%Ð%ØÔÐ+Ð+Ð+ØÔ×Ò˜wÑ'Ô'Ð'åÐ.°Ô0EÑFÔFð 	ÝðI¥$ t¡*¤*Ô"5ð Ið Ið Iñô ð õ ÐYÐYÐYÐYÀDÄJ×DVÒDVÑDXÔDXÐYÑYÔYÑYÔYð 	ÝðhØœÔ0ðhð hð hñô ð õ
 	(¨¬
°DÔ4DÑEÔEÐEàÔ*×9Ò9¸$¼*ÑEÔEˆŒ
Ø×ÒÑÔÐà�Š‰ŒˆàŒ=Ô�yÔ0Ò0Ð0Ø×!Ò! 'Ñ*Ô*Ð*Ø×#Ò#Ñ%Ô%Ð%ØŒ=Ô�yÔ0Ò0Ð0Ý! $¤/°4Ô3CÑDÔDÐDÐDÐDð 1Ð0rF   c                 ón   •— |                       ¦   «          t          ¦   «                              |¦  «        S rL   )Ú _reset_optimizers_and_schedulersr7   r„   )rC   rk   rD   s     €rE   r„   z&ModelParallelStrategy.setup_optimizers¾   s/   ø€ ð
 	×-Ò-Ñ/Ô/Ð/å‰wŒw×'Ò'¨Ñ0Ô0Ð0rF   c                 óV   — | j         €J ‚| j                              | j        ¦  «         d S rL   )rw   ÚtorO   rI   s    rE   r   z%ModelParallelStrategy.model_to_deviceÇ   s-   € àŒzÐ%Ð%Ð%ØŒ
�Š�dÔ&Ñ'Ô'Ð'Ð'Ð'rF   Ú
empty_init)NNNc              #   óò   K  — |rt          j        d¦  «        nt          ¦   «         }|5  | j                             ¦   «         5  d V — d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         d S # 1 swxY w Y   d S )NÚmeta)ÚtorchÚdevicer   r}   Útensor_init_context)rC   r‹   Úempty_init_contexts      rE   r�   z)ModelParallelStrategy.tensor_init_contextÌ   s  è è € ð 6@ÐR�Uœ\¨&Ñ1Ô1Ð1Å[Á]Ä]ÐØð 	ð 	 Ô!6×!JÒ!JÑ!LÔ!Lð 	ð 	ØˆEˆEˆEð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	s5   ©A,ÁAÁA,ÁA	ÁA,ÁA	ÁA,Á,A0Á3A0Únamec                 ó  — t          ¦   «         sd S t          j                             ¦   «         dk    r4t          j                             |                      ¦   «         ¬¦  «         d S t          j                             ¦   «          d S )NÚnccl)Ú
device_ids)r   rŽ   ÚdistributedÚget_backendr€   Ú_determine_device_ids)rC   r’   s     rE   r€   zModelParallelStrategy.barrierÔ   sv   € å*Ñ,Ô,ð 	ØˆFÝÔ×(Ò(Ñ*Ô*¨fÒ4Ð4ÝÔ×%Ò%°×1KÒ1KÑ1MÔ1MÐ%ÑNÔNÐNÐNÐNåÔ×%Ò%Ñ'Ô'Ð'Ð'Ð'rF   r   ÚobjÚsrcc                 ó�   — t          ¦   «         s|S |g}t          j                             ||t          j        ¬¦  «         |d         S )Nr   r   )r   rŽ   r–   Úbroadcast_object_listÚ_groupÚWORLD)rC   r™   rš   s      rE   Ú	broadcastzModelParallelStrategy.broadcastÝ   sE   € å*Ñ,Ô,ð 	ØˆJàˆeˆÝÔ×/Ò/°°SÅÄÐ/ÑMÔMÐMØ�1ŒvˆrF   ÚmeanÚtensorr   Ú	reduce_opc                 óT   — t          |t          ¦  «        rt          |||¬¦  «        S |S )N)r¢   )rr   r   r   )rC   r¡   r   r¢   s       rE   ÚreducezModelParallelStrategy.reduceæ   s1   € õ �f�fÑ%Ô%ð 	NÝ)¨&°%À9ÐMÑMÔMÐMØˆrF   c                 ó   — | j         j        gS rL   )rO   ÚindexrI   s    rE   r˜   z+ModelParallelStrategy._determine_device_idsñ   s   € ØÔ Ô&Ð'Ð'rF   c                 óÀ   — | j         €J ‚| j        €J ‚| j                              ¦   «          | j                             ¦   «          | j                             ¦   «          d S rL   )rb   rx   Úteardownr}   rI   s    rE   r¨   zModelParallelStrategy.teardownô   se   € àÔ'Ð3Ð3Ð3ØÔÐ+Ð+Ð+ØÔ ×)Ò)Ñ+Ô+Ð+ØÔ×&Ò&Ñ(Ô(Ð(ØÔ×!Ò!Ñ#Ô#Ð#Ð#Ð#rF   c                 ón   — ddl m}m}  || j         d¬¦  «        }| j        €J ‚ || j        |¬¦  «        S )z�Collects the state dict of the model.

        Only returns a non-empty state dict on rank 0 if ``save_distributed_checkpoint=False``.

        r   )ÚStateDictOptionsÚget_model_state_dictT©Úfull_state_dictÚcpu_offloadN©Úoptions)Ú'torch.distributed.checkpoint.state_dictrª   r«   r>   rw   )rC   rª   r«   Ústate_dict_optionss       rE   Úlightning_module_state_dictz1ModelParallelStrategy.lightning_module_state_dictü   sa   € ð 	cÐbÐbÐbÐbÐbÐbÐbà-Ð-À4ÔCdÐ?dÐswÐxÑxÔxÐØŒzÐ%Ð%Ð%Ø#Ð# D¤JÐ8JÐKÑKÔKÐKrF   Ú
checkpointÚstrictc                 ó   — d S rL   r]   )rC   r´   rµ   s      rE   Úload_model_state_dictz+ModelParallelStrategy.load_model_state_dict	  ó	   € ð 	ˆrF   Ú	optimizerc                 óT  — ddl m}m} ddlm} ddlm}  || j         d¬¦  «        }t          |t          ¦  «        r|j	        }| j
        €J ‚ || j
        ||¬¦  «        }| j        sA| j        dk    r6t          || j
        ¦  «        }|                     ||j        | j
        ¦  «        }|S )	z•Collects the state of the given optimizer.

        Only returns a non-empty state dict on rank 0 if ``save_distributed_checkpoint=False``.

        r   )rª   Úget_optimizer_state_dictrn   )ÚOptimStateKeyTypeTr¬   Nr¯   )r±   rª   r»   rv   ro   r¼   r>   rr   r%   Ú
_optimizerrw   Úglobal_rankÚ'_align_compiled_param_names_with_moduleÚrekey_optim_state_dictÚPARAM_ID)rC   r¹   rª   r»   ÚFSDPr¼   r²   Ú
state_dicts           rE   Úoptimizer_statez%ModelParallelStrategy.optimizer_state  sð   € ð 	gÐfÐfÐfÐfÐfÐfÐfØKÐKÐKÐKÐKÐKØ<Ð<Ð<Ð<Ð<Ð<à-Ð-À4ÔCdÐ?dÐswÐxÑxÔxÐÝ�iÕ!3Ñ4Ô4ð 	-Ø!Ô,ˆIàŒzÐ%Ð%Ð%à-Ð-¨d¬j¸)ÐM_Ð`Ñ`Ô`ˆ
ØÔ0ð 	i°TÔ5EÈÒ5JÐ5JÝ@ÀÈTÌZÑXÔXˆJØ×4Ò4°ZÐARÔA[Ð]aÔ]gÑhÔhˆJØÐrF   c                 ó   — d S rL   r]   )rC   r´   s     rE   Úload_optimizer_state_dictz/ModelParallelStrategy.load_optimizer_state_dict%  r¸   rF   ÚfilepathÚstorage_optionsc           	      óv  •— |�:t          dt          | ¦  «        j        › dt          | ¦  «        j        › d�¦  «        ‚t          |                      |¦  «        ¦  «        }|                     ¦   «         r(| j        s!t          |¦  «        st          d|› �¦  «        ‚| j        rÒ| 	                    ¦   «         r| 
                    ¦   «          |                     dd¬¦  «         d|                     d¦  «        i}|                     d„ t          |                     d	g ¦  «        ¦  «        D ¦   «         ¦  «         t          ||¦  «         | j        d
k    rt#          j        ||t&          z  ¦  «         d S d S t          |¦  «        rt)          j        |¦  «         t-          ¦   «                              ||¬¦  «        S )Nú`zF.save_checkpoint(..., storage_options=...)` is not supported because `z"` does not use the `CheckpointIO`.z/The checkpoint path exists and is a directory: T)ÚparentsÚexist_okrÃ   c                 ó    — i | ]\  }}d |› �|“ŒS ©Ú
optimizer_r]   )rs   ÚidxÚoptim_states      rE   ú
<dictcomp>z9ModelParallelStrategy.save_checkpoint.<locals>.<dictcomp>>  s6   € ð $ð $ð $á$�C˜ð #˜SÐ"Ð" Kð$ð $ð $rF   Úoptimizer_statesr   )r´   rÇ   )rz   r:   r;   r   rŸ   Úis_dirr>   r   ÚIsADirectoryErrorÚis_fileÚunlinkÚmkdirÚpopÚupdateÚ	enumerater   r¾   rŽ   Úsaver    ÚshutilÚrmtreer7   Úsave_checkpoint)rC   r´   rÇ   rÈ   ÚpathÚconverted_staterD   s         €rE   rß   z%ModelParallelStrategy.save_checkpoint*  sß  ø€ ð Ð&ÝðM•D˜‘J”JÔ'ð Mð MÝ˜$‘Z”ZÔ(ðMð Mð Mñô ð õ
 �D—N’N 8Ñ,Ô,Ñ-Ô-ˆØ�;Š;‰=Œ=ð 	^ Ô!Bð 	^ÕKaÐbfÑKgÔKgð 	^Ý#Ð$\ÐVZÐ$\Ð$\Ñ]Ô]Ð]àÔ,ð 	QØ�|Š|‰~Œ~ð Ø—’‘”�Ø�JŠJ˜t¨dˆJÑ3Ô3Ð3à+¨Z¯^ª^¸LÑ-IÔ-IÐJˆOØ×"Ò"ð $ð $å(1°*·.².ÐASÐUWÑ2XÔ2XÑ(YÔ(Yð$ñ $ô $ñ ô ð õ )¨¸$Ñ?Ô?Ð?àÔ 1Ò$Ð$Ý”
˜: tÕ.@Ñ'@ÑAÔAÐAÐAÐAð %Ð$õ & dÑ+Ô+ð $Ý”˜dÑ#Ô#Ð#Ý‘7”7×*Ò*°jÈ4Ð*ÑPÔPÐPrF   Úcheckpoint_pathÚweights_onlyc                 óâ   — t          |                      |¦  «        ¦  «        }d| j        id„ t          | j        ¦  «        D ¦   «         ¥}| j        €J ‚t          ||| j        j        d|¬¦  «        S )NrÃ   c                 ó    — i | ]\  }}d |› �|“ŒS rÎ   r]   )rs   rÐ   r¹   s      rE   rÒ   z9ModelParallelStrategy.load_checkpoint.<locals>.<dictcomp>Q  s'   € Ð^Ð^Ð^±°°iÐ!˜CÐ!Ð! 9Ð^Ð^Ð^rF   T)rà   r�   rµ   Úoptimizer_states_from_listrã   )r   rŸ   rw   rÛ   r†   rj   r   Ústrict_loading)rC   râ   rã   rà   r�   s        rE   Úload_checkpointz%ModelParallelStrategy.load_checkpointK  sˆ   € õ �D—N’N ?Ñ3Ô3Ñ4Ô4ˆà˜$œ*ð
à^Ð^Å9ÈTÌ_ÑC]ÔC]Ð^Ñ^Ô^ð
ˆð Ô$Ð0Ð0Ð0ÝØØØÔ(Ô7Ø'+Ø%ð
ñ 
ô 
ð 	
rF   c                 óX  •— t          ¦   «                              ¦   «          t          ¦   «          |                      ¦   «          |                      ¦   «         | _        | j        €J ‚d| j        i}t          r| j	        j
        dk    r| j	        nd |d<   t          | j        | j        fi |¤Ž d S )Nr3   ÚcpuÚ	device_id)r7   rg   r"   Úset_world_ranksÚ_get_process_group_backendr?   rb   r@   r   rO   r:   r   )rC   ÚkwargsrD   s     €rE   rh   z(ModelParallelStrategy._setup_distributed\  s´   ø€ Ý‰Œ×!Ò!Ñ#Ô#Ð#Ý‰ŒˆØ×ÒÑÔÐØ&*×&EÒ&EÑ&GÔ&GˆÔ#ØÔ'Ð3Ð3Ð3Ø"+¨T¬]Ð!;ˆÝ#ð 	_Ø6:Ô6FÔ6KÈuÒ6TÐ6T $Ô"2Ð"2ÐZ^ˆF�;ÑÝ˜dÔ6¸Ô8SÐ^Ð^ÐW]Ð^Ð^Ð^Ð^Ð^rF   c                 ó8   — | j         pt          | j        ¦  «        S rL   )r?   r   rO   rI   s    rE   rí   z0ModelParallelStrategy._get_process_group_backendg  s   € ØÔ*ÐmÕ.[Ð\`Ô\lÑ.mÔ.mÐmrF   c                 óú   — | j         �V| j                              | j        | j        z  | j        z   ¦  «         | j                              | j        | j        z  ¦  «         | j        xt          _	        t          _	        d S rL   )rb   Úset_global_rankÚ	node_rankrR   rN   Úset_world_sizerB   r¾   r   rV   Úutils_rank_zero_onlyrI   s    rE   rì   z%ModelParallelStrategy.set_world_ranksj  sr   € ØÔ#Ð/ØÔ$×4Ò4°T´^ÀdÔFXÑ5XÐ[_Ô[jÑ5jÑkÔkÐkØÔ$×3Ò3°D´NÀTÔEWÑ4WÑXÔXÐXð ;?Ô:JÐJ�ÔÕ2Ô7Ð7Ð7rF   )r4   r+   )r4   N)rk   rl   r4   NrL   )r   )Nr    )T)8r;   Ú
__module__Ú__qualname__Ú__doc__r   r   r   ÚintÚboolr   Ústrr   r8   ÚpropertyrJ   r   rŽ   r�   rO   rR   Údictr
   rZ   r2   r^   r`   re   rg   ry   r„   r   r   r   r�   r€   r(   rŸ   r   r$   r¤   Úlistr˜   r¨   r³   r   r·   r   rÄ   rÆ   r#   rß   rè   rh   rí   rì   Ú__classcell__)rD   s   @rE   r-   r-   <   s#  ø€ € € € € ðð ð, ;AØ<BØ,0Ø/3Ø'9ðð à! '¨&¤/°3Ð"6Ô7ðð $ G¨F¤O°SÐ$8Ô9ðð &*ð	ð
  (¨œ}ðð ˜)Ô$ðð 
ðð ð ð ð ð ð& ð!ð !ð !ñ „Xð!ð
 Øð6˜Uœ\ð 6ð 6ð 6ñ „Xñ „Xð6ð ðV˜sð Vð Vð Vñ „XðVð Øðh¨D°°c°¬Nð hð hð hñ „Xñ „Xðhð
 ð+ x°¤}ð +ð +ð +ñ „Xð+ð Øð°ð ð ð ñ „Xñ „Xðð Øð¨Tð ð ð ñ „Xñ „Xðð ðuð uð uñ „Xðuð
 ð?ð ?ð ?ð ?ð ?ñ „Xð?ð ðEð Eð Eñ „XðEð> ð1ð 1ð 1ð 1ð 1ñ „Xð1ð ð(ð (ð (ñ „Xð(ð Øðð ¨h°t¬nð È	ÐRbÔHcð ð ð ñ „Xñ „^ðð ð(ð (˜H SœMð (°Tð (ð (ð (ñ „Xð(ð ðð ˜Zð ¨cð ¸*ð ð ð ñ „Xðð ð  $Ø4:ð	ð à�f˜c�kÔ"ðð ˜Œ}ðð ˜E (¨C -Ô0Ô1ð	ð
 
ðð ð ñ „Xðð( t¨C¤yð (ð (ð (ð (ð ð$ð $ð $ñ „Xð$ð ð
L¨T°#°s°(¬^ð 
Lð 
Lð 
Lñ „Xð
Lð ðð °¸¸S¸Ô0Að È4ð Ð[_ð ð ð ñ „Xðð ð¨ð °t¸CÀ¸H´~ð ð ð ñ „Xðð, ð°G¸CÀ¸HÔ4Eð È$ð ð ð ñ „Xðð à\`ðQð QØ˜s C˜xœ.ðQØ49ðQØLTÐUXÌMðQà	ðQð Qð Qð Qð Qñ „XðQð@ ð
ð 
¨uð 
ÀHÈTÄNð 
Ð^bÐcfÐhkÐckÔ^lð 
ð 
ð 
ñ „Xð
ð 	_ð 	_ð 	_ð 	_ð 	_ð 	_ðn¨Cð nð nð nð nðKð Kð Kð Kð Kð Kð Kð KrF   r-   rÃ   Úmoduler4   c                 óô  ‡	‡
— ddl m} g Š
|                     ¦   «         D ]*\  }}t          ||¦  «        r‰
                     |¦  «         Œ+‰
s| S ‰
                     t          d¬¦  «         dt          dt          fˆ
fd„Š	d| v r.ˆ	fd	„| d                              ¦   «         D ¦   «         }i | ¥d|i¥} d
| v rDg }| d
         D ]1}i |¥dˆ	fd„|d         D ¦   «         i¥}|                     |¦  «         Œ2i | ¥d
|i¥} | S )as  Align optimizer state dict keys with a module that may have compiled submodules.

    When ``torch.compile`` wraps a submodule, its parameters appear under ``_orig_mod``.
    For example, ``model.0.weight`` becomes ``model._orig_mod.0.weight``.  The optimizer
    state dict returned by ``get_optimizer_state_dict`` may not include the ``_orig_mod``
    prefix, causing a mismatch when ``rekey_optim_state_dict`` builds its mapping from
    ``module.named_parameters()``.

    This function inserts ``._orig_mod`` into the state dict keys where necessary so that
    they match the module's ``named_parameters()`` output.

    r   )ÚOptimizedModuleT)ÚkeyÚreverser  r4   c                 óŽ   •— ‰D ]@}| |k    s|                       |dz   ¦  «        r | t          |¦  «        d …         }|› d|› �c S ŒA| S )Nú.z
._orig_mod)Ú
startswithrQ   )r  ÚprefixÚsuffixÚcompiled_prefixess      €rE   Ú_transform_keyz?_align_compiled_param_names_with_module.<locals>._transform_keyŽ  si   ø€ Ø'ð 	5ð 	5ˆFà�fŠ}ˆ} §¢¨v¸©|Ñ <Ô <ˆ}Ø�S ™[œ[˜]˜]Ô+�à Ð4Ð4¨FÐ4Ð4Ð4Ð4Ð4ð ð ˆ
rF   r�   c                 ó.   •— i | ]\  }} ‰|¦  «        |“ŒS r]   r]   )rs   ÚkÚvr
  s      €rE   rÒ   z;_align_compiled_param_names_with_module.<locals>.<dictcomp>™  s)   ø€ ÐRÐRÐR©d¨a°�^�^ AÑ&Ô&¨ÐRÐRÐRrF   Úparam_groupsÚparamsc                 ó&   •— g | ]} ‰|¦  «        ‘ŒS r]   r]   )rs   Úpr
  s     €rE   ú
<listcomp>z;_align_compiled_param_names_with_module.<locals>.<listcomp>   s#   ø€ Ð,XÐ,XÐ,XÀ1¨^¨^¸AÑ->Ô->Ð,XÐ,XÐ,XrF   )	Útorch._dynamor  Únamed_modulesrr   ÚappendÚsortrQ   rú   Úitems)rÃ   rÿ   r  r’   Ú	submoduleÚ	new_stateÚnew_param_groupsr   Ú	new_groupr
  r	  s            @@rE   r¿   r¿   s  sŒ  øø€ ð .Ð-Ð-Ð-Ð-Ð-ð $&ÐØ!×/Ò/Ñ1Ô1ð +ð +‰ˆˆiÝ�i Ñ1Ô1ð 	+Ø×$Ò$ TÑ*Ô*Ð*øàð ØÐð ×Ò�s¨DÐÑ1Ô1Ð1ð�Cð ¥Cð ð ð ð ð ð ð �*ÐÐØRÐRÐRÐR°jÀÔ6I×6OÒ6OÑ6QÔ6QÐRÑRÔRˆ	Ø7˜
Ð7 G¨YÐ7Ð7ˆ
ð ˜Ð#Ð#ØÐØ Ô/ð 	/ð 	/ˆEØY˜5ÐY (Ð,XÐ,XÐ,XÐ,XÈÈhÌÐ,XÑ,XÔ,XÐYÐYˆIØ×#Ò# IÑ.Ô.Ð.Ð.ØE˜
ÐE NÐ4DÐEÐEˆ
àÐrF   )MrÝ   Úcollections.abcr   r   Ú
contextlibr   r   Údatetimer   Úpathlibr   Útypingr	   r
   r   r   r   rŽ   Ú"lightning_utilities.core.rank_zeror   rô   r   Útorch.optimr   Útyping_extensionsr   Úpytorch_lightningÚplÚ5lightning_fabric.plugins.collectives.torch_collectiver   Ú*lightning_fabric.strategies.model_parallelr   r   r   r   Ú&lightning_fabric.utilities.distributedr   r   r   r   r   r�   Ú"lightning_fabric.utilities.importsr   r   Úlightning_fabric.utilities.initr   Úlightning_fabric.utilities.loadr    Ú$lightning_fabric.utilities.optimizerr!   Úlightning_fabric.utilities.seedr"   Ú lightning_fabric.utilities.typesr#   r$   Ú pytorch_lightning.core.optimizerr%   Ú8pytorch_lightning.strategies.launchers.subprocess_scriptr&   Ú%pytorch_lightning.strategies.parallelr'   Ú%pytorch_lightning.strategies.strategyr(   Ú pytorch_lightning.trainer.statesr)   Ú)pytorch_lightning.utilities.model_helpersr*   Ú%pytorch_lightning.utilities.rank_zeroÚtorch.distributed.device_meshr+   r-   rü   rú   ÚnnÚModuler¿   r]   rF   rE   ú<module>r9     s%  ðð €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à €€€Ø UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø TÐ TÐ TÐ TÐ TÐ Tðð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ KÐ KÐ KÐ KÐ KÐ KØ >Ð >Ð >Ð >Ð >Ð >Ø FÐ FÐ FÐ FÐ FÐ FØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø BÐ BÐ BÐ BÐ BÐ BØ <Ð <Ð <Ð <Ð <Ð <Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø CÐ CÐ CÐ CÐ CÐ CØ @Ð @Ð @Ð @Ð @Ð @àð 9Ø8Ð8Ð8Ð8Ð8Ð8ðtKð tKð tKð tKð tKÐ,ñ tKô tKð tKðn	1¸¸SÀ#¸X¼ð 1ÐPUÔPXÔP_ð 1ÐdhÐilÐnqÐiqÔdrð 1ð 1ð 1ð 1ð 1ð 1rF   