§
    Ò¬¸j/3  ã                  óH  — d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ e
r&d dlmZ d dlmZ d dlm Z m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'  ed¬¦  «         G d„ d¦  «        ¦   «         Z(d!d„Z)d"d „Z*dS )#é    )ÚannotationsN)Ú	dataclass)Úpartial)Úperf_counter)ÚTYPE_CHECKINGÚAny)Úeprint)Úparse_version)Ú+_get_credentials_from_provider_expiry_aware)Ú0_extract_table_statistics_from_delta_add_actions)Úscan_parquet)ÚScanCastOptions)ÚSchema)Údatetime)Ú
DeltaTable)ÚDeletionFilesÚStorageOptionsDict)ÚNoPickleOption)ÚCredentialProviderBuilder©Ú	LazyFrameT)Úkw_onlyc                  ó¬   — e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<   d*d„Zddddddœd+d „Zd,d"„Zd-d$„Zd.d&„Z	d/d)„Z
dS )0ÚDeltaDatasetzDataset interface for Delta.zNoPickleOption[DeltaTable]Útable_ú
str | NoneÚ
table_uri_zint | str | datetime | NoneÚversionzStorageOptionsDict | NoneÚstorage_optionsz CredentialProviderBuilder | NoneÚcredential_provider_builderzdict[str, Any] | NoneÚdelta_table_optionsÚboolÚuse_pyarrowÚpyarrow_optionszbool | NoneÚrechunkÚreturnr   c                óh   — t          |                      ¦   «                              ¦   «         ¦  «        S )zFetch the schema of the table.)r   ÚtableÚschema©Úselfs    úh/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/polars/io/delta/_dataset.pyr)   zDeltaDataset.schema4   s$   € å�d—j’j‘l”l×)Ò)Ñ+Ô+Ñ,Ô,Ð,ó    N)Úexisting_resolved_version_keyÚlimitÚ
projectionÚfilter_columnsÚpyarrow_predicater.   r/   ú
int | Noner0   úlist[str] | Noner1   r2   útuple[LazyFrame, str] | Nonec               óð  ‡%‡&‡'— ddl Š&ddl}|j        j                             ¦   «         }|r(t          d| j        › d|› d|› d|› d| j        › �
¦  «         |                      ¦   «         Š'| j        �| j        n‰'                     ¦   «         }t          |¦  «        }	|�||	k    r|rt          d|	›d	�¦  «         dS | j        r˜ddl
}dd
lm}
  ‰'j        d(i | j        pi ¤Ž}d}|�3ddl}ddlm}m}m}m} ddlm}m}m} t3          |||||||||dœ¦  «        }t5          |j        j        j        j        ||||¬¦  «        } |
j        |j         |dd¬¦  «        |	fS ‰' !                    ¦   «         }tE          |j#        ¦  «        Š%|                       ¦   «         }tI          ˆ%fd„| %                    ¦   «         D ¦   «         ¦  «        }tM          ¦   «         }|rt          d¦  «         ‰' '                    ¦   «         }|  (                    ¦   «          )                    d¦  «        rd„ |D ¦   «         }|r5tM          ¦   «         |z
  }t          dtU          |¦  «        › d|d›d�¦  «         |�3tW           ‰&j,        ‰' -                    ¦   «         ¦  «        |||¬¦  «        nd}‰' .                    ¦   «         j/        }|duod|v }d}|raddl0} d}!tc          | j2        ¦  «        }"|"|!k     r5dd 3                    d„ |!D ¦   «         ¦  «        › d|"› d�}#ti          |#¦  «        ‚d)ˆ&ˆ'fd#„}$d$|$f}nd}tk          |tU          ‰%¦  «        dk    r|ndtU          ‰%¦  «        dk    tm          j7        ¦   «         d%d&| j8        | j9        ||¬'¦
  «
        |	fS )*zConstruct a LazyFrame scan.r   Nz*DeltaDataset: to_dataset_scan(): version: z	, limit: z, projection: z, filter_columns: z, use_pyarrow: z=DeltaDataset: to_dataset_scan(): early return (version_key = ú)r   )Ú
to_py_dateÚto_py_datetimeÚ
to_py_timeÚto_py_timedelta)ÚDateÚDatetimeÚDuration)Úpar<   r=   r>   r8   r9   r:   r;   )Ún_rowsÚ	predicateÚwith_columnsT)ÚpyarrowÚis_purec                ó$   •— i | ]\  }}|‰v ¯	||“ŒS © rF   )Ú.0ÚkÚvÚpartition_columnss      €r,   ú
<dictcomp>z0DeltaDataset.to_dataset_scan.<locals>.<dictcomp>‘   s*   ø€ ÐGÐGÐG‘d�a˜°Ð5FÐ0FÐ0FˆQ�Ð0FÐ0FÐ0Fr-   z5DeltaDataset: to_dataset_scan(): begin path expansionú	lakefs://c                ó:   — g | ]}|                      d d¦  «        ‘ŒS )rL   ús3://)Úreplace)rG   Úpaths     r,   ú
<listcomp>z0DeltaDataset.to_dataset_scan.<locals>.<listcomp>œ   s&   € ÐJÐJÐJ¸D�T—\’\ +¨wÑ7Ô7ÐJÐJÐJr-   zCDeltaDataset: to_dataset_scan(): native scan_parquet(): num_files: z, path expansion time: z.3fÚs)r1   r)   ÚverboseÚdeletionVectors)é   é   é   z5reading delta deletion vectors requires deltalake >= ú.c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ©N)Ústr)rG   rI   s     r,   ú	<genexpr>z/DeltaDataset.to_dataset_scan.<locals>.<genexpr>À   s(   è è € Ð,LÐ,L¸­S°©V¬VÐ,LÐ,LÐ,LÐ,LÐ,LÐ,Lr-   z, found Úrequested_pathsúpl.DataFramer&   c                óº   •— t          ‰¦  «        }|€: ‰j        dd gt          | ¦  «        z  id ‰j        ‰j        ¦  «        i¬¦  «        S t          | |¦  «        S )NÚselection_vector)r)   )Ú_fetch_deletion_vectorsÚ	DataFrameÚlenÚListÚBooleanÚ_extract_delta_deletion_vectors)r]   Údelta_deletion_vectorsÚplr(   s     €€r,   Ú_deletion_vector_callbackz?DeltaDataset.to_dataset_scan.<locals>._deletion_vector_callbackÅ   s|   ø€ õ *AÀÑ)GÔ)GÐ&Ø)Ð1Ø'˜2œ<Ø+¨d¨Vµc¸/Ñ6JÔ6JÑ-JÐKØ 2°G°B´G¸B¼JÑ4GÔ4GÐHðñ ô ð õ 7Ø#Ð%;ñô ð r-   zdelta-deletion-vectorÚinsertÚignore)	Úhive_schemaÚhive_partitioningÚcast_optionsÚmissing_columnsÚextra_columnsr   Úcredential_providerÚ_table_statisticsÚ_deletion_filesrF   )r]   r^   r&   r^   ):ÚpolarsÚpolars._utils.loggingÚ_utilsÚloggingrS   r	   r   r#   r(   r[   Ú(polars.io.pyarrow_dataset.anonymous_scanÚpolars.lazyframe.framer   Úto_pyarrow_datasetr$   rC   Úpolars._utils.convertr8   r9   r:   r;   Úpolars.datatypesr<   r=   r>   Úevalr   ÚioÚpyarrow_datasetÚanonymous_scanÚ_scan_pyarrow_dataset_implÚ_scan_python_functionr)   ÚmetadataÚsetrJ   r   Úitemsr   Ú	file_urisÚ	table_uriÚ
startswithrc   r   rb   Úget_add_actionsÚprotocolÚreader_featuresÚ	deltalaker
   Ú__version__ÚjoinÚImportErrorr   r   Ú_default_icebergr   r    )(r+   r.   r/   r0   r1   r2   rt   rS   r   Úversion_keyr   ÚdatasetÚpa_predicate_exprr?   r8   r9   r:   r;   r<   r=   r>   ÚfuncÚtable_mdr)   rl   Ú
start_timeÚpathsÚelapsedÚtable_statisticsr‹   Úhas_deletion_vectorsÚdeletion_filesrŒ   Údv_min_versionÚ	installedÚmsgri   rJ   rh   r(   s(                                        @@@r,   Úto_dataset_scanzDeltaDataset.to_dataset_scan8   s<  øøø€ ð 	ÐÐÐØ$Ð$Ð$Ð$à”-Ô'×/Ò/Ñ1Ô1ˆàð 	Ýð3Ø œLð3ð 3àð3ð 3ð  *ð3ð 3ð $2ð	3ð 3ð
 !%Ô 0ð3ð 3ñô ð ð —
’
‘”ˆØ"&¤,Ð":�$”,�,ÀÇÂÁÄˆÝ˜'‘l”lˆð *Ð5Ø-°Ò<Ð<àð ÝØWÀkÐWÐWÐWñô ð ð �4àÔð *	Ø;Ð;Ð;Ð;Ø8Ð8Ð8Ð8Ð8Ð8à.�eÔ.ÐNÐN°$Ô2FÐ2LÈ"ÐNÐNˆGà $ÐØ Ð,Ø$Ð$Ð$Ð$ðð ð ð ð ð ð ð ð ð ð ð ð FÐEÐEÐEÐEÐEÐEÐEÐEÐEå$(Ø%à Ø $Ø$,Ø$,Ø&0Ø*8Ø&0Ø+:ð	ð 	ñ%ô %Ð!õ Ø”	Ô)Ô8ÔSØØØ+Ø'ðñ ô ˆDð 3�9Ô2Ø” ¨d¸Dðñ ô àðð ð —>’>Ñ#Ô#ˆÝ Ô :Ñ;Ô;Ðà—’‘”ˆÝØGÐGÐGÐG˜fŸlšl™nœnÐGÑGÔGñ
ô 
ˆõ "‘^”^ˆ
àð 	LÝÐJÑKÔKÐKà—’Ñ!Ô!ˆà�>Š>ÑÔ×&Ò& {Ñ3Ô3ð 	KØJÐJÀEÐJÑJÔJˆEàð 	Ý"‘n”n zÑ1ˆGÝð7å! %™jœjð7ð 7ð )0Ð6ð7ð 7ð 7ñô ð ð Ð)õ =Ø�”˜U×2Ò2Ñ4Ô4Ñ5Ô5Ø-ØØð	ñ ô ð ð ð 	ð  Ÿ.š.Ñ*Ô*Ô:ˆà 4Ð'ÐPÐ,=ÀÐ,Pð 	ð 04ˆØð 	"ØÐÐÐà&ˆNÝ% iÔ&;Ñ<Ô<ˆIØ˜>Ò)Ð)ð*Ø$'§H¢HÐ,LÐ,L¸^Ð,LÑ,LÔ,LÑ$LÔ$Lð*ð *à&ð*ð *ð *ð õ
 " #Ñ&Ô&Ð&ðð ð ð ð ð ð ð (Ø)ðˆNˆNð
 "ˆNåØÝ'*Ð+<Ñ'=Ô'=ÀÒ'AÐ'A˜˜ÀtÝ!Ð"3Ñ4Ô4°qÒ8Ý(Ô9Ñ;Ô;Ø$Ø"Ø Ô0Ø $Ô @Ø.Ø*ð
ñ 
ô 
ð ðð 	r-   r[   c                ó�   — | j         €9| j                             ¦   «         €J ‚|                      ¦   «         j        | _         | j         S )zFetch the table URI.)r   r   Úgetr(   r‡   r*   s    r,   r‡   zDeltaDataset.table_uriê   s<   € àŒ?Ð"Ø”;—?’?Ñ$Ô$Ð0Ð0Ð0Ø"Ÿjšj™lœlÔ4ˆDŒOàŒÐr-   r   c                ó  — | j                              ¦   «         �€Sddlm} ddlm}m}m} |                     d¦  «         ddl	m
} | j        €J ‚i }| j        r,| j                             ¦   «         x}rt          |¦  «        pi } || j        | j        | j        €| j        �i | j        pi ¥|¥nd| j        ¬¦  «        }|                     ¦   «         }	|	j        |k    s|	j        |k    rd|	j        › d	|› d
|› �}
 ||
¦  «        ‚|	j        dk    rG|	j        �@h |	j        £                     |¦  «        }t-          |¦  «        dk    rd|› d�}
 ||
¦  «        ‚| j                              |¦  «         | j                              ¦   «         S )zFetch the DeltaTable object.Nr   )ÚDeltaProtocolError)ÚMAX_SUPPORTED_READER_VERSIONÚNOT_SUPPORTED_READER_VERSIONÚSUPPORTED_READER_FEATURESrT   )Ú_get_delta_lake_table)Ú
table_pathr   r   r!   z&The table's minimum reader version is z5 but polars delta scanner only supports version 1 or z with these reader features: é   z)The table has set these reader features: z= but these are not yet supported by the polars delta scanner.)r   r¡   Údeltalake.exceptionsr£   Údeltalake.tabler¤   r¥   r¦   ÚaddÚpolars.io.delta._utilsr§   r   r    Úbuild_credential_providerr   r   r   r!   rŠ   Úmin_reader_versionr‹   Ú
differencerc   r„   )r+   r£   r¤   r¥   r¦   r§   Úcredential_provider_credsÚproviderr(   Útable_protocolrž   Úmissing_featuress               r,   r(   zDeltaDataset.tableò   sV  € àŒ;�?Š?ÑÔÑ$Ø?Ð?Ð?Ð?Ð?Ð?ðð ð ð ð ð ð ð ð ð ð &×)Ò)Ð*;Ñ<Ô<Ð<àDÐDÐDÐDÐDÐDà”?Ð.Ð.Ð.à(*Ð%àÔ/ð Ø Ô<×VÒVÑXÔXÐX�ðõ @ÀÑIÔIÐOÈRð *ð *Ð)Øœ?Øœð Ô+Ð7ØÔ7ÐCð R˜Ô,Ð2°ÐQÐ7PÐQÐQð à$(Ô$<ð
ñ 
ô 
ˆEð #Ÿ^š^Ñ-Ô-ˆNð Ô1Ð4PÒPÐPØ!Ô4Ð8TÒTÐTðb¸^Ô=^ð bð bØKgðbð bð G`ðbð bð ð )Ð(¨Ñ-Ô-Ð-àÔ1°QÒ6Ð6Ø"Ô2Ð>à#D ^Ô%CÐ#D×#OÒ#OØ-ñ$ô $Ð õ Ð'Ñ(Ô(¨1Ò,Ð,ð VÐFVð  Vð  Vð  V�CØ,Ð,¨SÑ1Ô1Ð1àŒK�OŠO˜EÑ"Ô"Ð"àŒ{�ŠÑ Ô Ð r-   údict[str, Any]c                ó8   — |                       ¦   «          | j        S rZ   )r‡   Ú__dict__r*   s    r,   Ú__getstate__zDeltaDataset.__getstate__2  s   € Ø�ŠÑÔÐØŒ}Ðr-   ÚstateÚNonec                ó   — || _         d S rZ   )r·   )r+   r¹   s     r,   Ú__setstate__zDeltaDataset.__setstate__6  s   € ØˆŒˆˆr-   )r&   r   )r.   r   r/   r3   r0   r4   r1   r4   r2   r   r&   r5   )r&   r[   )r&   r   )r&   rµ   )r¹   rµ   r&   rº   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r)   rŸ   r‡   r(   r¸   r¼   rF   r-   r,   r   r      s#  € € € € € € à&Ð&à&Ð&Ð&Ñ&ØÐÐÑØ(Ð(Ð(Ñ(à.Ð.Ð.Ñ.ØAÐAÐAÑAØ.Ð.Ð.Ñ.àÐÐÑØ*Ð*Ð*Ñ*àÐÐÑð-ð -ð -ð -ð 59Ø Ø'+Ø+/Ø(,ðlð lð lð lð lð lðdð ð ð ð>!ð >!ð >!ð >!ð@ð ð ð ðð ð ð ð ð r-   r   r]   r^   rg   r&   c                ór  — | j         dt          j        ik    sJ ‚t          j        t          j        t          j        ¦  «        dœ}|                     |                     ¦   «         ¦  «        }|j         |k    sJ ‚t          j        dk    rdnd}|  	                    ¦   «          
                    t          j        d¦  «        j                             dd¦  «        j                             |¦  «        ¦  «                             | 	                    ¦   «          
                    t          j        d¦  «        j                             dd¦  «        j                             |¦  «        ¦  «        ddd	d	¬
¦  «                             dg¦  «                             ¦   «         }|j        t%          | ¦  «        k    sJ ‚|S )a  
    Extract the deletion_vectors for the provided requested_paths.

    Input requested_paths schema is "path": String.
    Output series schema is "selection_vector": List(Boolean), maintaining order.

    The selection_vector from deltalake is a keep-mask (True = keep).
    rP   )Úfilepathr`   Úwin32zfile://zfile:///z
^lakefs://rN   rÃ   Úleft)Úleft_onÚright_onÚhowÚmaintain_orderr`   )r)   rh   ÚStringrd   re   ÚselectÚkeysÚsysÚplatformÚlazyrB   Úcolr[   rO   Ústrip_prefixrŽ   Ú_collect_eagerÚheightrc   )r]   rg   Údelta_dv_schemaÚfile_prefixÚ	joined_dfs        r,   rf   rf   :  sƒ  € ð Ô! f­b¬iÐ%8Ò8Ð8Ð8Ð8å#%¤9Å"Ä'Í"Ì*ÑBUÔBUÐVÐV€OØ3×:Ò:¸?×;OÒ;OÑ;QÔ;QÑRÔRÐØ!Ô(¨OÒ;Ð;Ð;Ð;å"œ|¨wÒ6Ð6�)�)¸J€Kà×ÒÑÔß	ŠÝŒF�6‰NŒNÜ—’˜ wÑ/Ô/Ü—’˜kÑ*Ô*ñ

ô 

÷
 
ŠØ"×'Ò'Ñ)Ô)×6Ò6Ý”�zÑ"Ô"Ü—W’W˜\¨7Ñ3Ô3Ü—\’\ +Ñ.Ô.ñô ð
 ØØØ!ð 
ñ 


ô 


÷ 
ŠÐ#Ð$Ñ	%Ô	%ß	ŠÑ	Ô	ð' ð, Ô�s ?Ñ3Ô3Ò3Ð3Ð3Ð3àÐr-   r(   r   úpl.DataFrame | Nonec                ó  — ddl }|j        j                             ¦   «         }t	          j        |                      ¦   «         ¦  «        }|r*|j        dk    rt          dt          |¦  «        › �¦  «         t          |¦  «        dk    rdS |S )aT  
    Fetch the deletion_vectors, mapping file_uri to "deletion_vector".

    Schema: {"filepath": pl.String, "selection_vector": pl.List(pl.Boolean)}

    The selection_vector from deltalake is a keep-mask (True = keep), so
    the more accurate term would be "selection_vector".

    Returns None if the table has no deletion vectors.
    r   Nz0DeltaDataset: has deletion_vectors, file_count: )
ru   rv   rw   rS   rh   rb   Údeletion_vectorsrÓ   r	   rc   )r(   rt   rS   Údv_tables       r,   ra   ra   h  sŽ   € ð !Ð Ð Ð àŒmÔ#×+Ò+Ñ-Ô-€GåŒ|˜E×2Ò2Ñ4Ô4Ñ5Ô5€Hàð S�8”? QÒ&Ð&ÝÐQÅ#ÀhÁ-Ä-ÐQÐQÑRÔRÐRå
ˆ8�}„}˜ÒÐØˆtà€Or-   )r]   r^   rg   r^   r&   r^   )r(   r   r&   r×   )+Ú
__future__r   rÍ   Údataclassesr   Ú	functoolsr   Útimer   Útypingr   r   rt   rh   ru   r	   Úpolars._utils.variousr
   Ú.polars.io.cloud.credential_provider._providersr   r­   r   Úpolars.io.parquet.functionsr   Ú#polars.io.scan_options.cast_optionsr   Úpolars.schemar   r   rŒ   r   Úpolars._typingr   r   Úpolars.io.cloud._utilsr   Ú,polars.io.cloud.credential_provider._builderr   ry   r   r   rf   ra   rF   r-   r,   ú<module>rè      sÿ  ðØ "Ð "Ð "Ð "Ð "Ð "à 
€
€
€
Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %à Ð Ð Ð Ø (Ð (Ð (Ð (Ð (Ð (Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð TÐ SÐ SÐ SÐ SÐ SØ 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø  Ð  Ð  Ð  Ð  Ð  àð 1Ø!Ð!Ð!Ð!Ð!Ð!à$Ð$Ð$Ð$Ð$Ð$à@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ø5Ð5Ð5Ð5Ð5Ð5ØVÐVÐVÐVÐVÐVØ0Ð0Ð0Ð0Ð0Ð0ð €�4ÐÑÔðWð Wð Wð Wð Wñ Wô Wñ ÔðWðt+ð +ð +ð +ð\ð ð ð ð ð r-   