§
    “©ºjyx  ã                   óê  — d dl Z d dlZd dlmZmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlZd dl m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z< d dl8m=Z> d dl?m@Z@mAZA d dlBmCZC d dlDmEZEmFZF d dlGmHZH d dlImJZJ d dlKmLZLmMZM d dlNmOZO d dlPmQZQ d dlRmSZS d dlTmUZU d dlVmWZW d dlXmYZY d dlZm[Z[ d d l\m]Z] d d!l^m_Z_mZm`Z` erHd d"lambZb d d#lcmdZdmeZemfZf d d$lgmhZh eeieje                  eeekelgekf         ehf         Zmeefed%         f         Zn e jo        ep¦  «        Zq G d&„ d'eW¦  «        ZrdS )(é    N)Ú	GeneratorÚMapping)ÚcontextmanagerÚnullcontext)Ú	timedelta)ÚPath)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚUnion)Úrank_zero_only)ÚTensor)ÚModule)Ú	Optimizer)Úoverride)ÚCheckpointIOÚClusterEnvironment)Údefault_pg_timeout)Ú_StrategyRegistry)Ú_METADATA_FILENAMEÚ _activation_checkpointing_kwargsÚ_auto_wrap_policy_kwargsÚ_distributed_checkpoint_loadÚ_distributed_checkpoint_saveÚ_get_full_state_dict_contextÚ_get_sharded_state_dict_contextÚ_init_cpu_offloadÚ_init_sharding_strategyÚ_is_full_checkpointÚ_is_sharded_checkpointÚ_move_torchmetrics_to_deviceÚ_optimizer_has_flat_paramsÚ_setup_activation_checkpointing)Ú_load_raw_module_state)Ú_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_TORCH_GREATER_EQUAL_2_2Ú_TORCH_GREATER_EQUAL_2_3)Ú&_has_meta_device_parameters_or_buffers)Ú
_lazy_loadÚ_materialize_tensors)Ú_optimizers_to_device)Ú
reset_seed)Ú_PATHÚReduceOp)ÚLightningOptimizer)Ú	Precision)ÚFSDPPrecision)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú
TBroadcast)Ú	TrainerFn)Úis_overridden)Úrank_zero_infor   Úrank_zero_warn)Ú
DeviceMesh)Ú
CPUOffloadÚMixedPrecisionÚShardingStrategy)ÚModuleWrapPolicy)Ú
FULL_SHARDÚSHARD_GRAD_OPÚNO_SHARDÚHYBRID_SHARDc            #       ó   ‡ — e Zd ZU dZdZg Zee         ed<   dddddde	ddddddddfde
d         d	e
eej                          d
e
e         de
e         de
e         de
e         de
e         deeddf         de
d         de
d         de
eee         eee                  f                  de
d         ddded         de
eeeef         df                  deddf"ˆ fd„Zeedej        fd „¦   «         ¦   «         Zedefd!„¦   «         Zede
e         fd"„¦   «         Zede
d         fd#„¦   «         Zeede fd$„¦   «         ¦   «         Z!e!j"        ede
e         ddfd%„¦   «         ¦   «         Z!eede#fd&„¦   «         ¦   «         Z$eedefd'„¦   «         ¦   «         Z%eedefd(„¦   «         ¦   «         Z&edVˆ fd)„¦   «         Z'defd*„Z(dVd+„Z)edVd,„¦   «         Z*ed-edefd.„¦   «         Z+edWd1„¦   «         Z,edWˆ fd2„¦   «         Z-edVd3„¦   «         Z.e/edXd4e
e         de0d5         fd6„¦   «         ¦   «         Z1e/ede0d5         fd7„¦   «         ¦   «         Z2edXd8e
e         ddfd9„¦   «         Z3edYd;e4d<ede4fd=„¦   «         Z5e	 	 dZd?ee6ef         d@e
e         dAe
ee7ef                  de6fdB„¦   «         Z8dee         fdC„Z9edVdD„¦   «         Z:e;dee         fdE„¦   «         Z<e;edFe=ddfdG„¦   «         ¦   «         Z>ede#eef         fdH„¦   «         Z?ed[dJe@eef         dKeddfdL„¦   «         ZAedMeBde#ee6f         fdN„¦   «         ZCedJe@eef         ddfdO„¦   «         ZDe	 dXdJe#eef         dPeEdQe
e         ddfˆ fdR„¦   «         ZFedXdSeEdTe
e         de#eef         fdU„¦   «         ZGˆ xZHS )\ÚFSDPStrategyae  Strategy for Fully Sharded Data Parallel provided by torch.distributed.

    Fully Sharded Training shards the entire model across all available GPUs, allowing you to scale model
    size, whilst using efficient communication to reduce overhead. In practice, this means we can remain
    at parity with PyTorch DDP, whilst scaling our model sizes dramatically. The technique is similar
    to ZeRO-Stage 3.

    For more information check out
    `this blogpost <https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api>`__.

    Defaults have been set and options have been exposed, but may require configuration
    based on your level of memory/speed efficiency. We suggest having a look at
    `this tutorial <https://pytorch.org/tutorials/intermediate/FSDP_tutorial.html>`__ for more information.

    Arguments:
        cpu_offload: See ``cpu_offload`` parameter in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.
        mixed_precision: See ``mixed_precision`` parameter in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.
        auto_wrap_policy: Same as ``auto_wrap_policy`` parameter in
            :class:`torch.distributed.fsdp.FullyShardedDataParallel`. For convenience, this also accepts a set of the
            layer classes to wrap.
        activation_checkpointing: Deprecated. Use ``activation_checkpointing_policy``.
        activation_checkpointing_policy: Same as ``auto_wrap_policy`` parameter in
            :class:`torch.distributed.fsdp.FullyShardedDataParallel` but used when selecting the modules for which you
            want to enable activation checkpointing. Enabling this can free up a significant amount of memory at the
            cost of speed since activations in these layers need to be recomputed during backpropagation. For
            convenience, this also accepts a set of the layer classes to wrap.
        sharding_strategy: Select whether to shard model parameters, gradients, optimizer states, or a combination of
            them. Available values are:

            - ``"FULL_SHARD"``: Shards model parameters, gradients, and optimizer states (default).
            - ``"SHARD_GRAD_OP"``: Shards gradients and optimizer states only. Model parameters get replicated.
            - ``"NO_SHARD"``: No sharding (identical to regular DDP).
            - ``"HYBRID_SHARD"``: Shards model parameters, gradients, and optimizer states within a single machine, but
              replicates across machines. See also the `device_mesh` parameter below.

            Also accepts a :class:`torch.distributed.fsdp.ShardingStrategy` enum value.

        device_mesh: A tuple `(replication size, sharding size)` that defines over how many devices to shard and
            replicate the model. The product of the two numbers must equal the world size. Only valid in combination
            with the `HYBRID_SHARD` sharding strategy.

        state_dict_type: The format in which the state of the model and optimizers gets saved into the checkpoint.

            - ``"full"``: The full weights and optimizer states get assembled on rank 0 and saved to a single file.
            - ``"sharded"``: Each rank saves its shard of weights and optimizer states to a file. The checkpoint is
              a folder with as many files as the world size.

        \**kwargs: See available parameters in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.

    ÚfsdpÚ_registered_strategiesNrE   ÚfullÚacceleratorzpl.accelerators.AcceleratorÚparallel_devicesÚcluster_environmentÚcheckpoint_ioÚprecision_pluginÚprocess_group_backendÚtimeoutÚcpu_offloadrA   Úmixed_precisionrB   Úauto_wrap_policyÚ_POLICYÚactivation_checkpointingÚactivation_checkpointing_policyÚsharding_strategyÚ_SHARDING_STRATEGYÚstate_dict_type)rM   ÚshardedÚdevice_meshr@   ÚkwargsÚreturnc                 óÄ  •— t          ¦   «                              |||||¬¦  «         d| _        || _        || _        t          |¦  «        | _        |	| _        t          |
|¦  «        | _	        |� t          st          d¦  «        ‚|| j	        d<   t          || j	        ¦  «        | _        | j	                             dd¦  «         t          ||¦  «        | _        || _        d S )N)rN   rO   rP   rQ   rR   é   z=The `device_mesh` argument is only supported in torch >= 2.2.r_   Úuse_orig_paramsT)ÚsuperÚ__init__Ú	num_nodesÚ_process_group_backendÚ_timeoutr   rU   rV   r   r`   r-   Ú
ValueErrorr    r[   Ú
setdefaultr   Ú_state_dict_type)ÚselfrN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rY   rZ   r[   r]   r_   r`   Ú	__class__s                    €úq/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/strategies/fsdp.pyrf   zFSDPStrategy.__init__“   sù   ø€ õ& 	‰Œ×ÒØ#Ø-Ø 3Ø'Ø-ð 	ñ 	
ô 	
ð 	
ð ˆŒØ&;ˆÔ#Ø-4ˆŒÝ,¨[Ñ9Ô9ˆÔØ.ˆÔÝ.Ð/?ÀÑHÔHˆŒàÐ"Ý+ð bÝ Ð!`ÑaÔaÐaØ)4ˆDŒK˜Ñ&å!8Ð9JÈDÌKÑ!XÔ!XˆÔð 	Œ×ÒÐ0°$Ñ7Ô7Ð7å0PØ$Ð&Eñ1
ô 1
ˆÔ-ð !0ˆÔÐÐó    c                 ó8   — | j         €J ‚| j         | j                 S ©N)rO   Ú
local_rank©rm   s    ro   Úroot_devicezFSDPStrategy.root_deviceÄ   s#   € ð Ô$Ð0Ð0Ð0ØÔ$ T¤_Ô5Ð5rp   c                 ó<   — | j         �t          | j         ¦  «        ndS )Nr   )rO   Úlenrt   s    ro   Únum_processeszFSDPStrategy.num_processesÊ   s!   € à-1Ô-BÐ-N�s�4Ô(Ñ)Ô)Ð)ÐTUÐUrp   c                 ó   — | j         S rr   )rh   rt   s    ro   rS   z"FSDPStrategy.process_group_backendÎ   s   € àÔ*Ð*rp   c                 óh   — | j         r| j         S | j        }t          |t          ¦  «        r|j        S d S rr   )rV   rR   Ú
isinstancer8   Úmixed_precision_config©rm   Úplugins     ro   r|   z#FSDPStrategy.mixed_precision_configÒ   s>   € àÔð 	(ØÔ'Ð'ØÔ&ˆÝ�f�mÑ,Ô,ð 	1ØÔ0Ð0Øˆtrp   c                 ód   — | j         }|�t          |t          ¦  «        sJ ‚|S t          d¦  «        S )Nz32-true)Ú_precision_pluginr{   r8   r}   s     ro   rR   zFSDPStrategy.precision_pluginÛ   s;   € ð Ô'ˆØÐÝ˜f¥mÑ4Ô4Ð4Ð4Ð4ØˆMÝ˜YÑ'Ô'Ð'rp   c                 óf   — |�'t          |t          ¦  «        st          d|› �¦  «        ‚|| _        d S )NzGThe FSDP strategy can only work with the `FSDPPrecision` plugin, found )r{   r8   Ú	TypeErrorr€   )rm   rR   s     ro   rR   zFSDPStrategy.precision_pluginä   sH   € ð Ð'µ
Ð;KÍ]Ñ0[Ô0[Ð'ÝØlÐZjÐlÐlñô ð ð "2ˆÔÐÐrp   c                 ó0   — | j         | j        z  | j        dœS )N)Únum_replicasÚrank)rg   rx   Úglobal_rankrt   s    ro   Údistributed_sampler_kwargsz'FSDPStrategy.distributed_sampler_kwargsí   s   € ð "&¤°$Ô2DÑ!DÈtÔO_Ð`Ð`Ð`rp   c                 ó   — dS )NT© rt   s    ro   Úrestore_checkpoint_after_setupz+FSDPStrategy.restore_checkpoint_after_setupò   s	   € ð ˆtrp   c                 ó   — dS )NFr‰   rt   s    ro   Úlightning_restore_optimizerz(FSDPStrategy.lightning_restore_optimizer÷   s	   € ð ˆurp   c                 óN  •— t          ¦   «                              ¦   «          t                               | j        j        › d�¦  «         t          ¦   «          |                      ¦   «          |                      ¦   «         | _	        | j
        €J ‚d| j        i}t          r| j        j        dk    r| j        nd |d<   t          | j
        | j	        fi |¤Ž t!          | j                             d¦  «        t&          ¦  «        r'ddlm}  |d| j        d         ¦  «        | j        d<   d S d S )	Nz: setting up distributed...rT   ÚcpuÚ	device_idr_   r   )Úinit_device_meshÚcuda)re   Úsetup_environmentÚlogÚdebugrn   Ú__name__r3   Úset_world_ranksÚ_get_process_group_backendrh   rP   ri   r.   ru   Útyper)   r{   r`   ÚgetÚtupleÚtorch.distributed.device_meshr�   )rm   r`   r�   rn   s      €ro   r’   zFSDPStrategy.setup_environmentü   s7  ø€ å‰Œ×!Ò!Ñ#Ô#Ð#Ý�	Š	�T”^Ô,ÐIÐIÐIÑJÔJÐJÝ‰Œˆð 	×ÒÑÔÐà&*×&EÒ&EÑ&GÔ&GˆÔ#ØÔ'Ð3Ð3Ð3Ø"+¨T¬]Ð!;ˆÝ#ð 	_Ø6:Ô6FÔ6KÈuÒ6TÐ6T $Ô"2Ð"2ÐZ^ˆF�;ÑÝ˜dÔ6¸Ô8SÐ^Ð^ÐW]Ð^Ð^Ð^õ �d”k—o’o mÑ4Ô4µeÑ<Ô<ð 	^ØFÐFÐFÐFÐFÐFà)9Ð)9¸&À$Ä+ÈmÔB\Ñ)]Ô)]ˆDŒK˜Ñ&Ð&Ð&ð	^ð 	^rp   c                 ó8   — | j         pt          | j        ¦  «        S rr   )rh   r(   ru   rt   s    ro   r—   z'FSDPStrategy._get_process_group_backend  s   € ØÔ*ÐmÕ.[Ð\`Ô\lÑ.mÔ.mÐmrp   c                 óú   — | j         �V| j                              | j        | j        z  | j        z   ¦  «         | j                              | j        | j        z  ¦  «         | j        xt          _	        t          _	        d S rr   )rP   Úset_global_rankÚ	node_rankrx   rs   Úset_world_sizerg   r†   r   r…   Úutils_rank_zero_onlyrt   s    ro   r–   zFSDPStrategy.set_world_ranks  sr   € ØÔ#Ð/ØÔ$×4Ò4°T´^ÀdÔFXÑ5XÐ[_Ô[jÑ5jÑkÔkÐkØÔ$×3Ò3°D´NÀTÔEWÑ4WÑXÔXÐXð ;?Ô:JÐJ�ÔÕ2Ô7Ð7Ð7rp   c                 ó~   — | j         €J ‚| j         j        s't          | j         | j        | j        ¦  «        | _        d S d S rr   )rP   Úcreates_processes_externallyr9   rx   rg   Ú	_launcherrt   s    ro   Ú_configure_launcherz FSDPStrategy._configure_launcher  sN   € àÔ'Ð3Ð3Ð3ØÔ'ÔDð 	uÝ6°tÔ7OÐQUÔQcÐeiÔesÑtÔtˆDŒNˆNˆNð	uð 	urp   Úmodelc           	      óö  ‡— ddl mŠ t          ˆfd„|                     ¦   «         D ¦   «         ¦  «        r?t	          |¦  «        rt          d¦  «         d| j        v rt          d¦  «         | j        d= n[t                               d| j	        j
        › d| j        › �¦  «          ‰d
|| j        | j        | j        | j	        j
        d	œ| j        ¤Ž}t          || j	        ¦  «         t          || j        ¦  «         |S )z|Wraps the model into a :class:`~torch.distributed.fsdp.fully_sharded_data_parallel.FullyShardedDataParallel`
        module.r   ©ÚFullyShardedDataParallelc              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S rr   )r{   )Ú.0Úmodr©   s     €ro   ú	<genexpr>z,FSDPStrategy._setup_model.<locals>.<genexpr>)  s.   øè è € ÐTÐT¸S�z˜#Ð7Ñ8Ô8ÐTÐTÐTÐTÐTÐTrp   zYThe model is already wrapped in `FSDP` but there are still parameters on the meta device.rW   z_A FSDP `auto_wrap_policy` is set, but the model is already wrapped. The policy will be ignored.z&setting up FSDP model with device id: z
, kwargs: )ÚmodulerU   rV   r[   r�   r‰   )Útorch.distributed.fsdpr©   ÚanyÚmodulesr/   r?   r`   r“   r”   ru   ÚindexrU   r|   r[   r#   r%   r   )rm   r¦   r©   s     @ro   Ú_setup_modelzFSDPStrategy._setup_model#  s>  ø€ ð 	DÐCÐCÐCÐCÐCåÐTÐTÐTÐTÀEÇMÂMÁOÄOÐTÑTÔTÑTÔTð 	Ý5°eÑ<Ô<ð ÝØoñô ð ð " T¤[Ð0Ð0åØuñô ð ð ”KÐ 2Ð3øå�IŠIÐn¸tÔ?OÔ?UÐnÐnÐaeÔalÐnÐnÑoÔoÐoØ,Ð,ð ØØ Ô,Ø $Ô ;Ø"&Ô"8ØÔ*Ô0ðð ð ”+ðð ˆEõ 	% U¨DÔ,<Ñ=Ô=Ð=õ 	(¨¨tÔ/TÑUÔUÐUàˆrp   Útrainerú
pl.Trainerc                 óÒ  — | j         €J ‚| j                              |¦  «         | j        €J ‚|j        j        t
          j        k    r+| j        r$| j                             | j        ¦  «        | _        | j	         
                    | j        ¦  «        | _        t          d| j        ¦  «        rt          d¦  «         n|                      | j        ¦  «        | _        |                      ¦   «          |j        j        t
          j        k    r|                      |¦  «         |                      ¦   «          |j        j        t
          j        k    rt%          | j        | j        ¦  «         d S d S )NÚconfigure_sharded_modelzÉYou have overridden `LightningModule.configure_sharded_model` hook. It will assume that all the layers are already wrapped for sharding and won't wrap the entire model using `FullyShardedDataParallel`.)rN   Úsetupr¦   ÚstateÚfnr<   ÚFITTINGÚ_layer_syncÚapplyrR   Úconvert_moduler=   Úlightning_moduler>   r³   ÚbarrierÚsetup_optimizersÚsetup_precision_pluginr2   Ú
optimizersru   )rm   r´   s     ro   r¸   zFSDPStrategy.setupF  sM  € àÔÐ+Ð+Ð+ØÔ×Ò˜wÑ'Ô'Ð'àŒzÐ%Ð%Ð%ØŒ=Ô�yÔ0Ò0Ð0°TÔ5EÐ0ØÔ)×/Ò/°´
Ñ;Ô;ˆDŒJàÔ*×9Ò9¸$¼*ÑEÔEˆŒ
åÐ2°DÔ4IÑJÔJð 	7åðvñô ð ð ð
 ×*Ò*¨4¬:Ñ6Ô6ˆDŒJØ�Š‰ŒˆàŒ=Ô�yÔ0Ò0Ð0Ø×!Ò! 'Ñ*Ô*Ð*Ø×#Ò#Ñ%Ô%Ð%ØŒ=Ô�yÔ0Ò0Ð0Ý! $¤/°4Ô3CÑDÔDÐDÐDÐDð 1Ð0rp   c                 ó¤  •— |                       ¦   «          | j                             d¦  «        r!t          ¦   «                              |¦  «        S d}	 t          ¦   «                              |¦  «         n+# t
          $ r}dt          |¦  «        vr‚ d}Y d }~nd }~ww xY w|st          d„ | j        D ¦   «         ¦  «        rt          d¦  «        ‚d S )Nrd   Fz%optimizer got an empty parameter listTc              3   ó6   K  — | ]}t          |¦  «         V — Œd S rr   )r$   )r«   Ú	optimizers     ro   r­   z0FSDPStrategy.setup_optimizers.<locals>.<genexpr>u  s/   è è € Ð&rÐ&rÐU^Õ+EÀiÑ+PÔ+PÐ'PÐ&rÐ&rÐ&rÐ&rÐ&rÐ&rrp   z×The optimizer does not seem to reference any FSDP parameters. HINT: Make sure to create the optimizer after setting up the model by referencing `self.trainer.model.parameters()` in the `configure_optimizers()` hook.)	Ú _reset_optimizers_and_schedulersr`   r™   re   rÁ   rj   Ústrr°   rÃ   )rm   r´   Úinvalid_params_errorÚexrn   s       €ro   rÁ   zFSDPStrategy.setup_optimizersa  s   ø€ ð
 	×-Ò-Ñ/Ô/Ð/àŒ;�?Š?Ð,Ñ-Ô-ð 	5Ý‘7”7×+Ò+¨GÑ4Ô4Ð4à$Ðð	(õ ‰GŒG×$Ò$ WÑ-Ô-Ð-Ð-øÝð 	(ð 	(ð 	(Ø6½cÀ"¹g¼gÐEÐEØØ#'Ð Ð Ð Ð Ð Ð øøøøð	(øøøð
  ð 	¥3Ð&rÐ&rÐbfÔbqÐ&rÑ&rÔ&rÑ#rÔ#rð 	åð2ñô ð ð
 ˆts   Á!A6 Á6
BÂ BÂBc                 ó   — d S rr   r‰   rt   s    ro   Úmodel_to_devicezFSDPStrategy.model_to_device~  ó	   € ð 	ˆrp   Ú
empty_init)NNNc              #   óò   K  — |rt          j        d¦  «        nt          ¦   «         }|5  | j                             ¦   «         5  d V — d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         d S # 1 swxY w Y   d S )NÚmeta)ÚtorchÚdevicer   rR   Útensor_init_context)rm   rÎ   Úempty_init_contexts      ro   rÓ   z FSDPStrategy.tensor_init_contextƒ  s  è è € ð 6@ÐR�Uœ\¨&Ñ1Ô1Ð1Å[Á]Ä]ÐØð 	ð 	 Ô!6×!JÒ!JÑ!LÔ!Lð 	ð 	ØˆEˆEˆEð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	s5   ©A,ÁAÁA,ÁA	ÁA,ÁA	ÁA,Á,A0Á3A0c           	   #   ó  K  — t                                | j        j        › d�¦  «         ddlm} ddlm}  |d|| j        | j	        | j
        | j        j        dœ| j        ¤Ž5  d V — d d d ¦  «         d S # 1 swxY w Y   d S )Nz : entered model_sharded_context.r   r¨   )Úenable_wrap)Úwrapper_clsrU   rV   r[   r�   r‰   )r“   r”   rn   r•   Ú2torch.distributed.fsdp.fully_sharded_data_parallelr©   Útorch.distributed.fsdp.wraprÖ   rU   r|   r[   ru   r²   r`   )rm   r©   rÖ   s      ro   Úmodel_sharded_contextz"FSDPStrategy.model_sharded_context�  s  è è € õ 	�	Š	�T”^Ô,ÐNÐNÐNÑOÔOÐOØ_Ð_Ð_Ð_Ð_Ð_Ø;Ð;Ð;Ð;Ð;Ð;àˆ[ð 
Ø0ØÔ(Ø Ô7Ø"Ô4ØÔ&Ô,ð
ð 
ð Œkð
ð 
ð 	ð 	ð ˆEˆEˆEð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	s   Á"A4Á4A8Á;A8Únamec                 ó  — t          ¦   «         sd S t          j                             ¦   «         dk    r4t          j                             |                      ¦   «         ¬¦  «         d S t          j                             ¦   «          d S )NÚnccl)Ú
device_ids)r'   rÑ   ÚdistributedÚget_backendrÀ   Ú_determine_device_ids)rm   rÛ   s     ro   rÀ   zFSDPStrategy.barrierž  sv   € å*Ñ,Ô,ð 	ØˆFÝÔ×(Ò(Ñ*Ô*¨fÒ4Ð4ÝÔ×%Ò%°×1KÒ1KÑ1MÔ1MÐ%ÑNÔNÐNÐNÐNåÔ×%Ò%Ñ'Ô'Ð'Ð'Ð'rp   r   ÚobjÚsrcc                 ó�   — t          ¦   «         s|S |g}t          j                             ||t          j        ¬¦  «         |d         S )Nr+   r   )r'   rÑ   rß   Úbroadcast_object_listÚ_groupÚWORLD)rm   râ   rã   s      ro   Ú	broadcastzFSDPStrategy.broadcast§  sE   € å*Ñ,Ô,ð 	ØˆJàˆeˆÝÔ×/Ò/°°SÅÄÐ/ÑMÔMÐMØ�1Œvˆrp   ÚmeanÚtensorr,   Ú	reduce_opc                 óT   — t          |t          ¦  «        rt          |||¬¦  «        S |S )a  Reduces a tensor from several distributed processes to one aggregated tensor.

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to gather results from. Defaults to all processes (world)
            reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
                Can also be a string 'sum' to calculate the sum during reduction.

        Return:
            reduced value, except when the input was not a tensor the output remains is unchanged

        )rë   )r{   r   r*   )rm   rê   r,   rë   s       ro   ÚreducezFSDPStrategy.reduce°  s1   € õ& �f�fÑ%Ô%ð 	NÝ)¨&°%À9ÐMÑMÔMÐMØˆrp   c                 ó   — | j         j        gS rr   )ru   r²   rt   s    ro   rá   z"FSDPStrategy._determine_device_idsÇ  s   € ØÔ Ô&Ð'Ð'rp   c                 óÔ  — t                                | j        j        › d�¦  «         | j        }|�Z|j        �S|j        j        j        t          j	        k    r4| j
        r-| j        €J ‚| j
                             | j        ¦  «        | _        | j        €J ‚| j        €J ‚| j                             ¦   «          | j                             ¦   «          | j                             ¦   «          d S )Nz: tearing down strategy...)r“   r”   rn   r•   r¿   Ú_trainerr¹   rº   r<   r»   r¼   r¦   ÚrevertrP   rN   ÚteardownrR   )rm   Ú	pl_modules     ro   rò   zFSDPStrategy.teardownÊ  së   € å�	Š	�T”^Ô,ÐHÐHÐHÑIÔIÐIàÔ)ˆ	àÐ!ð Ô"Ð.ØÔ"Ô(Ô+­yÔ/@Ò@Ð@ØÔ ð Að ”:Ð)Ð)Ð)ØÔ)×0Ò0°´Ñ<Ô<ˆDŒJàÔ'Ð3Ð3Ð3ØÔÐ+Ð+Ð+ØÔ ×)Ò)Ñ+Ô+Ð+ØÔ×&Ò&Ñ(Ô(Ð(ØÔ×!Ò!Ñ#Ô#Ð#Ð#Ð#rp   c                 ó   — | j         S rr   )rL   )Úclss    ro   Úget_registered_strategiesz&FSDPStrategy.get_registered_strategiesà  s   € àÔ)Ð)rp   Ústrategy_registryc                 ó  — t           j                             ¦   «         sd S |                     d| d¬¦  «         | j                             d¦  «         |                     d| dd¬¦  «         | j                             d¦  «         d S )NrK   z+Fully Sharded Data Parallel (FSDP) training)ÚdescriptionÚfsdp_cpu_offloadzQFully Sharded Data Parallel (FSDP) training with Full Sharding and CPU OffloadingT)rù   rU   )rÑ   rß   Úis_availableÚregisterrL   Úappend)rõ   r÷   s     ro   Úregister_strategiesz FSDPStrategy.register_strategiesä  s¨   € õ Ô ×-Ò-Ñ/Ô/ð 	ØˆFØ×"Ò"ØØØEð 	#ñ 	
ô 	
ð 	
ð
 	Ô"×)Ò)¨&Ñ1Ô1Ð1à×"Ò"ØØØkØð	 	#ñ 	
ô 	
ð 	
ð 	Ô"×)Ò)Ð*<Ñ=Ô=Ð=Ð=Ð=rp   c                 ó:  — | j         €J ‚| j        dk    rt          | j         ¦  «        }n>| j        dk    rt          | j         | j        ¬¦  «        }nt          d| j        › �¦  «        ‚|5  | j                              ¦   «         cd d d ¦  «         S # 1 swxY w Y   d S )Nr^   rM   ©Ú
world_sizeúUnknown state_dict_type: )r¦   rl   r   r   r  rj   Ú
state_dict)rm   Ústate_dict_ctxs     ro   Úlightning_module_state_dictz(FSDPStrategy.lightning_module_state_dictø  sì   € àŒzÐ%Ð%Ð%ØÔ  IÒ-Ð-Ý<¸T¼ZÑHÔHˆNˆNØÔ" fÒ,Ð,Ý9¸$¼*ÐQUÔQ`ÐaÑaÔaˆNˆNåÐP¸Ô9NÐPÐPÑQÔQÐQØð 	+ð 	+Ø”:×(Ò(Ñ*Ô*ð	+ð 	+ð 	+ð 	+ñ 	+ô 	+ð 	+ð 	+ð 	+ð 	+ð 	+ð 	+øøøð 	+ð 	+ð 	+ð 	+ð 	+ð 	+s   Á*BÂBÂBTÚ
checkpointÚstrictc                 ó   — d S rr   r‰   )rm   r  r  s      ro   Úload_model_state_dictz"FSDPStrategy.load_model_state_dict  rÍ   rp   rÆ   c                 óF  — ddl m} ddl m} t          |t          ¦  «        r|j        }| j        €J ‚| j        dk    rHt          | j        ¦  «        5  | 	                    | j        |¦  «        cd d d ¦  «         S # 1 swxY w Y   n‡| j        dk    r|t          | j        | j        ¬¦  «        5  | 	                    | j        |¦  «        }| j        dk    r!|                     ||j        | j        ¦  «        }|cd d d ¦  «         S # 1 swxY w Y   t          d| j        › �¦  «        ‚)Nr   r¨   ©ÚOptimStateKeyTyper^   rM   r   r  )r¯   r©   r  r{   r6   Ú
_optimizerr¦   rl   r   Úoptim_state_dictr   r  r†   Úrekey_optim_state_dictÚPARAM_IDrj   )rm   rÆ   ÚFSDPr  r  s        ro   Úoptimizer_statezFSDPStrategy.optimizer_state	  sØ  € àKÐKÐKÐKÐKÐKØ<Ð<Ð<Ð<Ð<Ð<å�iÕ!3Ñ4Ô4ð 	-Ø!Ô,ˆIàŒzÐ%Ð%Ð%ØÔ  IÒ-Ð-Ý0°´Ñ<Ô<ð Dð DØ×,Ò,¨T¬Z¸ÑCÔCðDð Dð Dð Dñ Dô Dð Dð Dð Dð Dð Dð Døøøð Dð Dð Dð Dð Dð Ô" fÒ,Ð,Ý-¨d¬jÀTÄ_ÐUÑUÔUð "ð "Ø!×2Ò2°4´:¸yÑIÔI�
ØÔ# qÒ(Ð(à!%×!<Ò!<¸ZÐIZÔIcÐeiÔeoÑ!pÔ!p�JØ!ð"ð "ð "ð "ñ "ô "ð "ð "ð "ð "ð "ð "øøøð "ð "ð "ð "õ ÐL°TÔ5JÐLÐLÑMÔMÐMs%   ÁA9Á9A=Â A=Â+A	DÄDÄDc                 ó   — d S rr   r‰   )rm   r  s     ro   Úload_optimizer_state_dictz&FSDPStrategy.load_optimizer_state_dict   rÍ   rp   ÚfilepathÚstorage_optionsc           	      ót  •— |�t          d¦  «        ‚t          |                      |¦  «        ¦  «        }|                     ¦   «         r,| j        dk    r!t          |¦  «        st          d|› �¦  «        ‚| j        dk    rÒ|                     ¦   «         r|                     ¦   «          | 	                    dd¬¦  «         d| 
                    d¦  «        i}|                     d	„ t          | 
                    d
g ¦  «        ¦  «        D ¦   «         ¦  «         t          ||¦  «         | j        dk    rt          j        ||t"          z  ¦  «         d S d S | j        dk    rFt          |¦  «        rt%          j        |¦  «         t)          ¦   «                              ||¬¦  «        S t-          d| j        › �¦  «        ‚)Nz�`FSDPStrategy.save_checkpoint(..., storage_options=...)` is not supported because `FSDPStrategy` does not use the `CheckpointIO`.rM   z/The checkpoint path exists and is a directory: r^   T)ÚparentsÚexist_okr¦   r  c                 ó    — i | ]\  }}d |› �|“ŒS )Ú
optimizer_r‰   )r«   ÚidxÚoptim_states      ro   ú
<dictcomp>z0FSDPStrategy.save_checkpoint.<locals>.<dictcomp>9  s6   € ð $ð $ð $á$�C˜ð #˜SÐ"Ð" Kð$ð $ð $rp   Úoptimizer_statesr   )r  r  r  )r‚   r   rè   Úis_dirrl   r"   ÚIsADirectoryErrorÚis_fileÚunlinkÚmkdirÚpopÚupdateÚ	enumerater   r†   rÑ   Úsaver   ÚshutilÚrmtreere   Úsave_checkpointrj   )rm   r  r  r  ÚpathÚconverted_statern   s         €ro   r+  zFSDPStrategy.save_checkpoint%  sÑ  ø€ ð Ð&ÝðCñô ð õ
 �D—N’N 8Ñ,Ô,Ñ-Ô-ˆØ�;Š;‰=Œ=ð 	^˜TÔ2°fÒ<Ð<ÕE[Ð\`ÑEaÔEaÐ<Ý#Ð$\ÐVZÐ$\Ð$\Ñ]Ô]Ð]àÔ  IÒ-Ð-Ø�|Š|‰~Œ~ð Ø—’‘”�Ø�JŠJ˜t¨dˆJÑ3Ô3Ð3à&¨
¯ª°|Ñ(DÔ(DÐEˆOØ×"Ò"ð $ð $å(1°*·.².ÐASÐUWÑ2XÔ2XÑ(YÔ(Yð$ñ $ô $ñ ô ð õ
 )¨¸$Ñ?Ô?Ð?àÔ 1Ò$Ð$Ý”
˜: tÕ.@Ñ'@ÑAÔAÐAÐAÐAð %Ð$àÔ" fÒ,Ð,Ý% dÑ+Ô+ð $Ý”˜dÑ#Ô#Ð#Ý‘7”7×*Ò*°jÈ4Ð*ÑPÔPÐPåÐP¸Ô9NÐPÐPÑQÔQÐQrp   Úcheckpoint_pathÚweights_onlyc                 ó\  — t          |                      |¦  «        ¦  «        }ddlm} | j        €J ‚| j        €J ‚t          |¦  «        �rSddlm} t          | j        ¦  «        }|5  d| j         
                    ¦   «         i}t          ||¦  «         | j                             |d         | j        j        ¬¦  «         | j        j        j        j        t"          j        k    r„| j        r}ddlm}  ||¬¦  «        }	t-          | j        ¦  «        D ]V\  }
}d|
› �} ||d         ||	¬	¦  «        }|                     ||         | j        |¬
¦  «        }|                     |¦  «         ŒWd d d ¦  «         n# 1 swxY w Y   t1          j        |t4          z  |¬¦  «        }|S t7          |¦  «        �ràt9          |¦  «        }t;          |                     d¦  «        | j        | j        | j        j        ¬¦  «         tA          |¦  «        }ddlm} ddlm!} | "                    d¦  «        }|�$| j        j        j        j        t"          j        k    r|S tG          | j        ¦  «        tG          |¦  «        k    r5tI          dtG          | j        ¦  «        › dtG          |¦  «        › d�¦  «        ‚tK          | j        | j        d¬¦  «        5  tM          | j        |¦  «        D ]˜\  }}tO          tQ          |d          )                    ¦   «         ¦  «        d         tT          ¦  «        r!| +                    ||j,        | j        ¦  «        }|                     || j        |¬
¦  «        }|                     |¦  «         Œ™	 d d d ¦  «         n# 1 swxY w Y   |S t[          dt]          |¦  «        ›d�¦  «        ‚)Nr   r¨   )Ú!load_sharded_optimizer_state_dictr¦   )r  )ÚFileSystemReader)r,  r  )Úmodel_state_dictÚoptimizer_keyÚstorage_reader)r  r¦   Úoptim)r/  r  )r®   r  r  r  r  zYou have configured z( optimizers but the checkpoint contains z€ optimizers to load. Please resume training with the same number of optimizers or edit the checkpoint manually to remove states.F)r  Ú
rank0_onlyr¹   z	The path zœ does not point to a valid checkpoint. Make sure the path points to either a directory with FSDP checkpoint shards, or a single file with a full checkpoint.)/r   rè   r¯   r©   r¦   r¿   r"   Ú&torch.distributed.checkpoint.optimizerr1  r   r  r   Úload_state_dictÚstrict_loadingr´   r¹   rº   r<   r»   rÃ   Útorch.distributed.checkpointr2  r'  Úoptim_state_dict_to_loadrÑ   Úloadr   r!   r0   r&   r%  r  r1   r  r™   rw   ÚRuntimeErrorr   Úzipr{   ÚlistÚkeysÚintr  Ú
PARAM_NAMErj   rÈ   )rm   r.  r/  r,  r  r1  r  Úmodule_stater2  Úreaderr  r6  Ú	optim_keyr  Úflattened_osdÚmetadatar  r  r  rÆ   Ú	opt_states                        ro   Úload_checkpointzFSDPStrategy.load_checkpointI  s³  € õ �D—N’N ?Ñ3Ô3Ñ4Ô4ˆàKÐKÐKÐKÐKÐKàŒzÐ%Ð%Ð%ØÔ$Ð0Ð0Ð0å! $Ñ'Ô'ñ !	Ø`Ð`Ð`Ð`Ð`Ð`å<¸T¼ZÑHÔHˆNàð =ð =Ø '¨¬×)>Ò)>Ñ)@Ô)@ÐA�Ý,¨\¸4Ñ@Ô@Ð@Ø”
×*Ò*¨<¸Ô+@ÈÔI^ÔImÐ*ÑnÔnÐnàÔ(Ô0Ô6Ô9½YÔ=NÒNÐNÐSWÔSbÐNØMÐMÐMÐMÐMÐMð .Ð-°4Ð8Ñ8Ô8�Få&/°´Ñ&@Ô&@ð =ð =™
˜˜UØ$6°Ð$6Ð$6˜	Ø&GÐ&GØ-9¸'Ô-BØ*3Ø+1ð'ñ 'ô '˜ð
 )-×(EÒ(EØ-8¸Ô-CØ"&¤*Ø"'ð )Fñ )ô )˜ð
 ×-Ò-¨mÑ<Ô<Ð<Ð<ð1=ð =ð =ñ =ô =ð =ð =ð =ð =ð =ð =øøøð =ð =ð =ð =õ6 ”z $Õ);Ñ";È,ÐWÑWÔWˆHØˆOå˜tÑ$Ô$ñ )	Ý# DÑ)Ô)ˆJÝ"Ø—’˜|Ñ,Ô,Ø”zØœ?ØÔ,Ô;ð	ñ ô ð õ .¨jÑ9Ô9ˆJàOÐOÐOÐOÐOÐOØ@Ð@Ð@Ð@Ð@Ð@à)Ÿ~š~Ð.@ÑAÔAÐØÐ'¨4Ô+@Ô+HÔ+NÔ+QÕU^ÔUfÒ+fÐ+fà!Ð!Ý�4”?Ñ#Ô#¥sÐ+;Ñ'<Ô'<Ò<Ð<Ý"ðW­3¨t¬Ñ+?Ô+?ð Wð WÝÐ,Ñ-Ô-ðWð Wð Wñô ð õ .¨d¬jÀTÄ_ÐafÐgÑgÔgð 9ð 9Ý,/°´ÐAQÑ,RÔ,Rð 
9ð 
9Ñ(�I˜yÝ!¥$ y°Ô'9×'>Ò'>Ñ'@Ô'@Ñ"AÔ"AÀ!Ô"DÅcÑJÔJð uà$(×$?Ò$?À	ÐK\ÔKgÐimÔisÑ$tÔ$t˜	à $× =Ò =Ø)2Ø"œjØ'ð !>ñ !ô !�Ið
 ×-Ò-¨iÑ8Ô8Ð8Ð8ð
9ð9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9ð Ðåð_�˜D™	œ	ð _ð _ð _ñ
ô 
ð 	
s&   Á'D E3Å3E7Å:E7ËB/NÎNÎN)ra   N)r´   rµ   ra   Nrr   )r   )Nré   )T)Ir•   Ú
__module__Ú__qualname__Ú__doc__Ústrategy_namerL   r@  rÈ   Ú__annotations__r   r   rÑ   rÒ   r   r   r7   r   r   Úboolr˜   r   r   rš   rB  r
   rf   Úpropertyr   ru   rx   rS   r|   r8   rR   ÚsetterÚdictr‡   rŠ   rŒ   r’   r—   r–   r¥   r³   r¸   rÁ   rÌ   r   r   rÓ   rÚ   rÀ   r;   rè   r   r5   rí   rá   rò   Úclassmethodrö   r   rþ   r  r   r	  r   r  r  r4   r+  rJ  Ú__classcell__)rn   s   @ro   rJ   rJ   \   sX  ø€ € € € € € ð1ð 1ðf €MØ(*Ð˜D œIÐ*Ð*Ñ*ð @DØ9=Ø<@Ø04Ø04Ø/3Ø'9Ø7;Ø6:Ø04ØVZØ?CØ2>Ø6<ØFJð!/0ð /0àÐ;Ô<ð/0ð # 4¨¬Ô#5Ô6ð/0ð &Ð&8Ô9ð	/0ð
   Ô-ð/0ð # 9Ô-ð/0ð  (¨œ}ð/0ð ˜)Ô$ð/0ð ˜4 ¨tÐ3Ô4ð/0ð "Ð"2Ô3ð/0ð # 9Ô-ð/0ð #+¨5°°f´¸tÀDÈÄLÔ?QÐ1QÔ+RÔ"Sð/0ð *2°)Ô)<ð/0ð 0ð/0ð !Ð!2Ô3ð/0ð  ˜e E¨#¨s¨(¤O°\Ð$AÔBÔCð!/0ð" ð#/0ð$ 
ð%/0ð /0ð /0ð /0ð /0ð /0ðb Øð6˜Uœ\ð 6ð 6ð 6ñ „Xñ „Xð6ð ðV˜sð Vð Vð Vñ „XðVð ð+ x°¤}ð +ð +ð +ñ „Xð+ð ð¨Ð1AÔ(Bð ð ð ñ „Xðð Øð( -ð (ð (ð (ñ „Xñ „Xð(ð ÔØð2°¸)Ô1Dð 2Èð 2ð 2ð 2ñ „Xñ Ôð2ð Øða¨Dð að að añ „Xñ „Xðað Øð°ð ð ð ñ „Xñ „Xðð Øð¨Tð ð ð ñ „Xñ „Xðð ð^ð ^ð ^ð ^ð ^ñ „Xð^ð*n¨Cð nð nð nð nðKð Kð Kð Kð ðuð uð uñ „Xðuð
 ð  &ð  ¨Vð  ð  ð  ñ „Xð ðD ðEð Eð Eñ „XðEð4 ðð ð ð ð ñ „Xðð8 ðð ð ñ „Xðð Øðð ¨h°t¬nð È	ÐRbÔHcð ð ð ñ „Xñ „^ðð Øð yÐ1AÔ'Bð ð ð ñ „Xñ „^ðð ð(ð (˜H SœMð (°Tð (ð (ð (ñ „Xð(ð ðð ˜Zð ¨cð ¸*ð ð ð ñ „Xðð ð  $Ø4:ð	ð à�f˜c�kÔ"ðð ˜Œ}ðð ˜E (¨C -Ô0Ô1ð	ð
 
ðð ð ñ „Xðð,( t¨C¤yð (ð (ð (ð (ð ð$ð $ð $ñ „Xð$ð* ð*¨$¨s¬)ð *ð *ð *ñ „[ð*ð Øð>Ð4Eð >È$ð >ð >ð >ñ „Xñ „[ð>ð$ ð	+¨T°#°s°(¬^ð 	+ð 	+ð 	+ñ „Xð	+ð ðð °¸¸S¸Ô0Að È4ð Ð[_ð ð ð ñ „Xðð ðN¨ð N°t¸CÀ¸KÔ7Hð Nð Nð Nñ „XðNð, ð°G¸CÀ¸HÔ4Eð È$ð ð ð ñ „Xðð à\`ð!Rð !RØ˜s C˜xœ.ð!RØ49ð!RØLTÐUXÌMð!Rà	ð!Rð !Rð !Rð !Rð !Rñ „Xð!RðF ðZ
ð Z
¨uð Z
ÀHÈTÄNð Z
Ð^bÐcfÐhkÐckÔ^lð Z
ð Z
ð Z
ñ „XðZ
ð Z
ð Z
ð Z
ð Z
rp   rJ   )sÚloggingr)  Úcollections.abcr   r   Ú
contextlibr   r   Údatetimer   Úpathlibr   Útypingr	   r
   r   r   r   r   rÑ   Ú"lightning_utilities.core.rank_zeror   r¡   r   Útorch.nnr   Útorch.optimr   Útyping_extensionsr   Úpytorch_lightningÚplÚlightning_fabric.pluginsr   r   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Úlightning_fabric.strategiesr   Ú lightning_fabric.strategies.fsdpr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   Ú*lightning_fabric.strategies.model_parallelr&   Ú&lightning_fabric.utilities.distributedr'   r(   r)   r*   r,   ræ   Ú"lightning_fabric.utilities.importsr-   r.   Úlightning_fabric.utilities.initr/   Úlightning_fabric.utilities.loadr0   r1   Ú$lightning_fabric.utilities.optimizerr2   Úlightning_fabric.utilities.seedr3   Ú lightning_fabric.utilities.typesr4   r5   Ú pytorch_lightning.core.optimizerr6   Ú#pytorch_lightning.plugins.precisionr7   Ú(pytorch_lightning.plugins.precision.fsdpr8   Ú8pytorch_lightning.strategies.launchers.subprocess_scriptr9   Ú%pytorch_lightning.strategies.parallelr:   Ú%pytorch_lightning.strategies.strategyr;   Ú pytorch_lightning.trainer.statesr<   Ú)pytorch_lightning.utilities.model_helpersr=   Ú%pytorch_lightning.utilities.rank_zeror>   r?   r›   r@   rØ   rA   rB   rC   rÙ   rD   Úsetr˜   rP  rB  rX   r\   Ú	getLoggerr•   r“   rJ   r‰   rp   ro   ú<module>ry     s}  ðð €€€Ø €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð €€€Ø UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ TÐ TÐ TÐ TÐ TÐ TØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð  NÐ MÐ MÐ MÐ MÐ Mðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ RÐ RÐ RÐ RÐ RÐ RØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ FÐ FÐ FÐ FÐ FÐ FØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø BÐ BÐ BÐ BÐ BÐ BØ ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø BÐ BÐ BÐ BÐ BÐ BØ <Ð <Ð <Ð <Ð <Ð <Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø CÐ CÐ CÐ CÐ CÐ CØ `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `àð uØ8Ð8Ð8Ð8Ð8Ð8ØoÐoÐoÐoÐoÐoÐoÐoÐoÐoØ<Ð<Ð<Ð<Ð<Ð<à�C˜˜VœÔ% x°¸¸sÐ0CÀTÐ0IÔ'JÐL\Ð\Ô]€GØÐ/°Ð9rÔ1sÐsÔtÐð €gÔ˜Ñ!Ô!€ðH	
ð H	
ð H	
ð H	
ð H	
Ð#ñ H	
ô H	
ð H	
ð H	
ð H	
rp   