§
    “©ºj³,  ã                   ó"  — d dl Z d dlmZmZmZmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZ d dlmZ d dlmZ d dlmZmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'm(Z( erd dl)m*Z*  G d„ de¦  «        Z+dS )é    N)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚUnion)ÚTensor)ÚModule)Ú	Optimizer)Ú
DataLoader)Úoverride)ÚAccelerator)Ú_XLA_GREATER_EQUAL_2_1)ÚCheckpointIOÚ	PrecisionÚXLAPrecision)ÚXLAEnvironment)ÚXLACheckpointIO)ÚParallelStrategyÚ_StrategyRegistry)Ú_XLALauncher)Ú
TBroadcast)Úrank_zero_only)Ú_PATHÚReduceOp©ÚMpDeviceLoaderc                   óÒ  ‡ — e Zd ZdZ	 	 	 	 	 d5dee         deeej                          dee	         dee
         ded	dfˆ fd
„Zeed	ej        fd„¦   «         ¦   «         Zed	efd„¦   «         Zeed	e	fd„¦   «         ¦   «         Zej        edee         d	dfd„¦   «         ¦   «         Zeed	e
fd„¦   «         ¦   «         Zej        edee         d	dfd„¦   «         ¦   «         Zeed	efˆ fd„¦   «         ¦   «         Zeed	efˆ fd„¦   «         ¦   «         Zeed	efˆ fd„¦   «         ¦   «         Zeed	efˆ fd„¦   «         ¦   «         Zed6d„¦   «         Zed6ˆ fd„¦   «         Zeded	efd„¦   «         Zeded	dfd„¦   «         Zede d	dfd„¦   «         Z!ed7de"d ee#         d!ed	e"fd"„¦   «         Z$e	 d8d#e%e"e#f         d ee#         d$ee%e&e'f                  d	e"fd%„¦   «         Z(ed9d&ee'         d'e#d(e#d	dfd)„¦   «         Z)ed:d+e*d,ed	e*fd-„¦   «         Z+e	 	 d8d.e,d/e-e'e%ee.e#f         f         d0ee#         d1ee-e'e/e'e#gef         f                  d	df
ˆ fd2„¦   «         Z0e1ed3e2d	dfd4„¦   «         ¦   «         Z3ˆ xZ4S );ÚXLAStrategyzxStrategy for training multiple TPU devices using the :func:`torch_xla.distributed.xla_multiprocessing.spawn`
    method.NTÚacceleratorÚparallel_devicesÚcheckpoint_ioÚ	precisionÚsync_module_statesÚreturnc                 ó–   •— t          ¦   «                              ||t          ¦   «         ||¬¦  «         d | _        d| _        || _        d S )N)r   r    Úcluster_environmentr!   r"   F)ÚsuperÚ__init__r   Ú_backward_sync_controlÚ	_launchedÚ_sync_module_states)Úselfr   r    r!   r"   r#   Ú	__class__s         €úo/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/strategies/xla.pyr(   zXLAStrategy.__init__+   sZ   ø€ õ 	‰Œ×ÒØ#Ø-Ý .Ñ 0Ô 0Ø'Øð 	ñ 	
ô 	
ð 	
ð '+ˆÔ#ØˆŒØ#5ˆÔ Ð Ð ó    c                 óh   — | j         st          d¦  «        ‚dd lmc m} |                     ¦   «         S )NzFAccessing the XLA device before processes have spawned is not allowed.r   )r*   ÚRuntimeErrorÚtorch_xla.core.xla_modelÚcoreÚ	xla_modelÚ
xla_device)r,   Úxms     r.   Úroot_devicezXLAStrategy.root_device>   sG   € ð Œ~ð 	iÝÐgÑhÔhÐhØ-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-à�}Š}‰ŒÐr/   c                 ó<   — | j         �t          | j         ¦  «        ndS ©Nr   )r    Úlen©r,   s    r.   Únum_processeszXLAStrategy.num_processesG   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUr/   c                 ób   — | j         }|�t          |t          ¦  «        sJ ‚|S t          ¦   «         S ©N)Ú_checkpoint_ioÚ
isinstancer   ©r,   Úplugins     r.   r!   zXLAStrategy.checkpoint_ioK   s9   € ð Ô$ˆØÐÝ˜f¥oÑ6Ô6Ð6Ð6Ð6ØˆMÝÑ Ô Ð r/   Úioc                 óf   — |�'t          |t          ¦  «        st          d|› �¦  «        ‚|| _        d S )NzHThe XLA strategy can only work with the `XLACheckpointIO` plugin, found )r@   r   Ú	TypeErrorr?   )r,   rC   s     r.   r!   zXLAStrategy.checkpoint_ioT   s=   € ð ˆ>¥*¨RµÑ"AÔ"Aˆ>ÝÐkÐgiÐkÐkÑlÔlÐlØ ˆÔÐÐr/   c                 ód   — | j         }|�t          |t          ¦  «        sJ ‚|S t          d¦  «        S )Nz32-true)Ú
_precisionr@   r   rA   s     r.   r"   zXLAStrategy.precision[   s:   € ð ”ˆØÐÝ˜f¥lÑ3Ô3Ð3Ð3Ð3ØˆMÝ˜IÑ&Ô&Ð&r/   c                 óf   — |�'t          |t          ¦  «        st          d|› �¦  «        ‚|| _        d S )NzEThe XLA strategy can only work with the `XLAPrecision` plugin, found )r@   r   rE   rG   )r,   r"   s     r.   r"   zXLAStrategy.precisiond   s<   € ð Ð ­°I½|Ñ)LÔ)LÐ ÝÐoÐdmÐoÐoÑpÔpÐpØ#ˆŒˆˆr/   c                 ó<   •— | j         rt          ¦   «         j        ndS r9   )r*   r'   Úglobal_rank©r,   r-   s    €r.   rJ   zXLAStrategy.global_rankk   s   ø€ ð '+¤nÐ;�u‰wŒwÔ"Ð"¸!Ð;r/   c                 ó<   •— | j         rt          ¦   «         j        ndS r9   )r*   r'   Ú
local_rankrK   s    €r.   rM   zXLAStrategy.local_rankp   ó   ø€ ð &*¤^Ð:�u‰wŒwÔ!Ð!¸Ð:r/   c                 ó<   •— | j         rt          ¦   «         j        ndS r9   )r*   r'   Ú	node_rankrK   s    €r.   rP   zXLAStrategy.node_ranku   s   ø€ ð %)¤NÐ9�u‰wŒwÔ Ð ¸Ð9r/   c                 ó<   •— | j         rt          ¦   «         j        ndS )Né   )r*   r'   Ú
world_sizerK   s    €r.   rS   zXLAStrategy.world_sizez   rN   r/   c                 ó.   — t          | ¦  «        | _        d S r>   )r   Ú	_launcherr;   s    r.   Ú_configure_launcherzXLAStrategy._configure_launcher   s   € å% dÑ+Ô+ˆŒˆˆr/   c                 ó  •— | j         €J ‚t          | j         ¦  «        dk    r%t          dt          | ¦  «        j        › d�¦  «        ‚d| _        | j        t          _        t          ¦   «          
                    ¦   «          d S )NrR   zThe zƒ does not support running on a single device with the PjRT runtime. Try using all devices or the `SingleDeviceXLAStrategy` strategyT)r    r:   ÚNotImplementedErrorÚtypeÚ__name__r*   rJ   r   Úrankr'   Úsetup_environmentrK   s    €r.   r\   zXLAStrategy.setup_environmentƒ   s�   ø€ àÔ$Ð0Ð0Ð0ÝˆtÔ$Ñ%Ô%¨Ò*Ð*õ &ðS•t˜D‘z”zÔ*ð Sð Sð Sñô ð ð
 ˆŒØ"Ô.�ÔÝ‰Œ×!Ò!Ñ#Ô#Ð#Ð#Ð#r/   Úmodulec                 óR   — | j         rt          rddlm} nddlm}  ||¦  «         |S )Nr   )Úbroadcast_master_param)r+   r   r2   r_   Útorch_xla.experimental.pjrt)r,   r]   r_   s      r.   Úsetup_modulezXLAStrategy.setup_module’   sS   € àÔ#ð 	+Ý%ð OØKÐKÐKÐKÐKÐKÐKàNÐNÐNÐNÐNÐNà"Ð" 6Ñ*Ô*Ð*àˆr/   c                 ó:   — |                      | j        ¦  «         d S r>   )Útor7   )r,   r]   s     r.   Úmodule_to_devicezXLAStrategy.module_to_devicež   s   € à�	Š	�$Ô"Ñ#Ô#Ð#Ð#Ð#r/   Ú
dataloaderr   c                 ó°   — ddl m} t          ||¦  «        r|S  ||| j        ¦  «        }|j        j        |_        t          |j        dd ¦  «        |_        |S )Nr   r   Úbatch_sampler)Ú%torch_xla.distributed.parallel_loaderr   r@   r7   Ú_loaderÚdatasetÚgetattrrg   )r,   re   r   s      r.   Úprocess_dataloaderzXLAStrategy.process_dataloader¢   sl   € àHÐHÐHÐHÐHÐHå�j .Ñ1Ô1ð 	àÐà#�^ J°Ô0@ÑAÔAˆ
à'Ô/Ô7ˆ
ÔÝ#*¨:Ô+=¸ÐPTÑ#UÔ#Uˆ
Ô ØÐr/   FÚtensorÚgroupÚ
sync_gradsc                 óÒ  — | j         s|S t          |t          ¦  «        s't          dt	          | ¦  «        j        › d|› �¦  «        ‚|                     ¦   «         dk    r|                     d¦  «        }|j        }| 	                    | j
        ¦  «        }ddlmc m} ddlmc m} |r|                     |¦  «        n|                     |¦  «        }| 	                    |¦  «        }|S )aC  Function to gather a tensor from several distributed processes.

        Args:
            tensor: tensor to all-gather.
            group: unused.
            sync_grads: flag that allows users to synchronize gradients for the all-gather operation.
        Return:
            A tensor of shape (world_size, ...)

        ú`z4.all_gather` is only implemented for tensors. Given r   N)r*   r@   r   rX   rY   rZ   ÚdimÚ	unsqueezeÚdevicerc   r7   Útorch_xla.core.functionsr3   Ú	functionsr2   r4   Ú
all_gather)r,   rm   rn   ro   Úoriginal_deviceÚxfr6   s          r.   rw   zXLAStrategy.all_gather°   s
  € ð Œ~ð 	ØˆMÝ˜&¥&Ñ)Ô)ð 	Ý%Øe•D˜‘J”JÔ'ÐeÐeÐ]cÐeÐeñô ð ð �:Š:‰<Œ<˜1ÒÐØ×%Ò% aÑ(Ô(ˆFØ œ-ˆØ—’˜4Ô+Ñ,Ô,ˆà-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ø-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-à*4ÐO�—’˜vÑ&Ô&Ð&¸"¿-º-ÈÑ:OÔ:OˆØ—’˜?Ñ+Ô+ˆØˆr/   ÚoutputÚ	reduce_opc                 óæ  — t          |t          ¦  «        st          j        || j        ¬¦  «        }t          |t
          ¦  «        o|t
          j        k    }t          |t          ¦  «        o|                     ¦   «         dv}|s|rt          d|› �¦  «        ‚dd l
mc m} |                     d|t          ¦  «        }t          |t          ¦  «        r |                     ¦   «         dv r
|| j        z  }|S )N)rt   )ÚsumÚmeanÚavgz]Currently, the XLAStrategy only supports `sum`, `mean`, `avg` for the reduce operation, got: r   Úreduce)r   r~   )r@   r   Útorchrm   r7   r   ÚSUMÚstrÚlowerÚ
ValueErrorr2   r3   r4   Úmesh_reducer}   rS   )r,   rz   rn   r{   Úinvalid_reduce_opÚinvalid_reduce_op_strr6   s          r.   Ú
all_reducezXLAStrategy.all_reduceÎ   s  € õ ˜&¥&Ñ)Ô)ð 	CÝ”\ &°Ô1AÐBÑBÔBˆFå& yµ(Ñ;Ô;ÐYÀ	ÍXÌ\Ò@YÐÝ *¨9µcÑ :Ô :Ð n¸y¿ºÑ?PÔ?PÐXnÐ?nÐØð 	Ð 5ð 	Ýð Øð ð  ñô ð ð 	.Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-à—’ ¨&µ#Ñ6Ô6ˆå�i¥Ñ%Ô%ð 	.¨)¯/ª/Ñ*;Ô*;¸Ð*NÐ*NØ˜dœoÑ-ˆFàˆr/   ÚnameÚargsÚkwargsc                 ó\   — | j         sd S dd lmc m} |€d}|                     |¦  «         d S )Nr   Ú )r*   r2   r3   r4   Ú
rendezvous)r,   rŠ   r‹   rŒ   r6   s        r.   ÚbarrierzXLAStrategy.barrierå   sN   € àŒ~ð 	ØˆFØ-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-àˆ<àˆDØ
�Š�dÑÔÐÐÐr/   r   ÚobjÚsrcc                 ó   — | j         s|S dd lmc m} t	          |t
          ¦  «        }|rO|                     ¦   «         dk    r|                     d¦  «        }|j        }| 	                    | j
        ¦  «        }nmt          j        ¦   «         }t          j        ||¦  «         t          j        t!          |                     ¦   «         ¦  «        | j
        t          j        ¬¦  «        }|g}|                     ||¬¦  «         |d         }|s_t          j        |                     ¦   «                              ¦   «                              ¦   «         ¦  «        }t          j        |¦  «        }n| 	                    |¦  «        }|S )Nr   )rt   Údtype)Úroot_ordinal)r*   r2   r3   r4   r@   r   rr   rs   rt   rc   r7   rC   ÚBytesIOr�   Úsaverm   Ú	bytearrayÚ	getbufferÚfloatÚcollective_broadcastÚcpuÚbyteÚnumpyÚload)r,   r‘   r’   r6   Ú	is_tensorrx   Úbuffers          r.   Ú	broadcastzXLAStrategy.broadcastð   s^  € àŒ~ð 	ØˆJà-Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-å˜s¥FÑ+Ô+ˆ	Øð 	Ø�wŠw‰yŒy˜AŠ~ˆ~Ø—m’m AÑ&Ô&�Ø!œjˆOà—&’&˜Ô)Ñ*Ô*ˆCˆCõ ”Z‘\”\ˆFÝŒJ�s˜FÑ#Ô#Ð#Ý”,Ý˜&×*Ò*Ñ,Ô,Ñ-Ô-°dÔ6FÍeÌkðñ ô ˆCð ˆeˆØ
×Ò °#ÐÑ6Ô6Ð6Ø�!Œfˆàð 	*å”Z §¢¡	¤	§¢Ñ 0Ô 0× 6Ò 6Ñ 8Ô 8Ñ9Ô9ˆFÝ”*˜VÑ$Ô$ˆCˆCà—&’&˜Ñ)Ô)ˆCàˆ
r/   ÚpathÚstateÚstorage_optionsÚfilterc                 óŒ   •— ddl mc m} |                     ¦   «          t	          ¦   «                              ||||¬¦  «         dS )a„  Save model, optimizer, and other state as a checkpoint file.

        Args:
            path: A path to where the file(s) should be saved
            state: A dictionary with contents to be saved. If the dict contains modules or optimizers, their
                state-dict will be retrieved and converted automatically.
            storage_options: Additional options for the ``CheckpointIO`` plugin
            filter: An optional dictionary of the same format as ``state`` mapping keys to callables that return a
                boolean indicating whether the given parameter should be saved (``True``) or filtered out (``False``).

        r   N)r¥   r¦   )r2   r3   r4   Ú	mark_stepr'   Úsave_checkpoint)r,   r£   r¤   r¥   r¦   r6   r-   s         €r.   r©   zXLAStrategy.save_checkpoint  sW   ø€ ð& 	.Ð-Ð-Ð-Ð-Ð-Ð-Ð-Ð-ð 	�Š‰Œˆå‰Œ×Ò  e¸_ÐU[ÐÑ\Ô\Ð\Ð\Ð\r/   Ústrategy_registryc                 ó@   — |                      d| | j        ¬¦  «         d S )NÚxla)Údescription)ÚregisterrZ   )Úclsrª   s     r.   Úregister_strategieszXLAStrategy.register_strategies-  s'   € ð 	×"Ò" 5¨#¸3¼<Ð"ÑHÔHÐHÐHÐHr/   )NNNNT)r$   N)NF)NNr>   )r   )5rZ   Ú
__module__Ú__qualname__Ú__doc__r   r   Úlistr�   rt   r   r   Úboolr(   Úpropertyr   r7   Úintr<   r!   Úsetterr   r"   r   rJ   rM   rP   rS   rV   r\   r	   ra   rd   r   rl   r   r   rw   r   r   rƒ   r‰   r�   r   r¢   r   Údictr
   r   r©   Úclassmethodr   r°   Ú__classcell__)r-   s   @r.   r   r   '   sÒ  ø€ € € € € ðð ð
 .2Ø9=Ø37Ø,0Ø#'ð6ð 6à˜kÔ*ð6ð # 4¨¬Ô#5Ô6ð6ð   Ô0ð	6ð
 ˜LÔ)ð6ð !ð6ð 
ð6ð 6ð 6ð 6ð 6ð 6ð& Øð˜Uœ\ð ð ð ñ „Xñ „Xðð ðV˜sð Vð Vð Vñ „XðVð Øð!˜ð !ð !ð !ñ „Xñ „Xð!ð ÔØð! ¨Ô 6ð !¸4ð !ð !ð !ñ „Xñ Ôð!ð
 Øð'˜<ð 'ð 'ð 'ñ „Xñ „Xð'ð ÔØð$ 8¨IÔ#6ð $¸4ð $ð $ð $ñ „Xñ Ôð$ð
 Øð<˜Sð <ð <ð <ð <ð <ñ „Xñ „Xð<ð Øð;˜Cð ;ð ;ð ;ð ;ð ;ñ „Xñ „Xð;ð Øð:˜3ð :ð :ð :ð :ð :ñ „Xñ „Xð:ð Øð;˜Cð ;ð ;ð ;ð ;ð ;ñ „Xñ „Xð;ð ð,ð ,ð ,ñ „Xð,ð ð$ð $ð $ð $ð $ñ „Xð$ð ð	 6ð 	¨fð 	ð 	ð 	ñ „Xð	ð ð$ vð $°$ð $ð $ð $ñ „Xð$ð ð¨Zð Ð<Lð ð ð ñ „Xðð ðð  ð °¸´ð ÐRVð Ðcið ð ð ñ „Xðð: àswðð Ø˜F C˜KÔ(ðØ19¸#´ðØRZÐ[`ÐaiÐknÐanÔ[oÔRpðà	ðð ð ñ „Xðð, ðð ˜H SœMð ¸ð Èð ÐPTð ð ð ñ „Xðð ð ð  ˜Zð  ¨cð  ¸*ð  ð  ð  ñ „Xð ðD ð
 *.ØBFð]ð ]àð]ð �C˜˜v y°#Ð5Ô6Ð6Ô7ð]ð " #œð	]ð
 ˜˜c 8¨S°#¨J¸Ð,<Ô#=Ð=Ô>Ô?ð]ð 
ð]ð ]ð ]ð ]ð ]ñ „Xð]ð2 ØðIÐ4Eð IÈ$ð Ið Ið Iñ „Xñ „[ðIð Ið Ið Ið Ir/   r   ),rC   Útypingr   r   r   r   r   r�   r   Útorch.nnr	   Útorch.optimr
   Útorch.utils.datar   Útyping_extensionsr   Úlightning_fabric.acceleratorsr   Ú!lightning_fabric.accelerators.xlar   Úlightning_fabric.pluginsr   r   r   Ú%lightning_fabric.plugins.environmentsr   Úlightning_fabric.plugins.io.xlar   Úlightning_fabric.strategiesr   r   Ú)lightning_fabric.strategies.launchers.xlar   Ú$lightning_fabric.strategies.strategyr   Ú$lightning_fabric.utilities.rank_zeror   Ú lightning_fabric.utilities.typesr   r   rh   r   r   © r/   r.   ú<module>rÌ      sÍ  ðð 
€	€	€	Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø &Ð &Ð &Ð &Ð &Ð &à 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø DÐ DÐ DÐ DÐ DÐ DØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ @Ð @Ð @Ð @Ð @Ð @Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ BÐ BÐ BÐ BÐ BÐ BØ ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <àð EØDÐDÐDÐDÐDÐDðIIð IIð IIð IIð IIÐ"ñ IIô IIð IIð IIð IIr/   