
    ITh@<              
          S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKJ	r	J
r
  S SKJr  S SKJrJrJrJrJrJr  SSKJrJrJr  \R0                  " \5      r1 Skr1 S	kr\	 " S
 S5      5       rS\\R<                  \4   S\\\4   4S jr S\\\R<                  4   S\\\\\\\!4   4      SS4S jr"S\\\R<                  4   S\\\R<                  4   SS4S jr#S\RH                  S\S\\\R<                  \!4   SS4S jr%S\\\\!4   S\!4S jr&S\S\4S jr'S\S\\   SS4S jr(S\RH                  S\RR                  S\*4S jr+g)     N)contextmanager)	dataclassfield)Path)AnyDict	GeneratorIterableTupleUnion   )DDUFCorruptedFileErrorDDUFExportErrorDDUFInvalidEntryNameError>   .txt.json.model.safetensors>   config.jsonscheduler_config.jsontokenizer_config.jsonpreprocessor_config.jsonc                       \ rS rSr% Sr\\S'   \\S'   \\S'   \" SS9r	\
\S'   \S	\\S
S
4   4S j5       rSS\S	\4S jjrSrg
)	DDUFEntry"   a  Object representing a file entry in a DDUF file.

See [`read_dduf_file`] for how to read a DDUF file.

Attributes:
    filename (str):
        The name of the file in the DDUF archive.
    offset (int):
        The offset of the file in the DDUF archive.
    length (int):
        The length of the file in the DDUF archive.
    dduf_path (str):
        The path to the DDUF archive (for internal use).
filenamelengthoffsetF)repr	dduf_pathreturnNc              #   \  #    U R                   R                  S5       n[        R                  " UR                  5       S[        R                  S9 nX R
                  U R
                  U R                  -    v   SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f7f)zOpen the file as a memory-mapped file.

Useful to load safetensors directly from the file.

Example:
    ```py
    >>> import safetensors.torch
    >>> with entry.as_mmap() as mm:
    ...     tensors = safetensors.torch.load(mm)
    ```
rbr   )r   accessN)r    openmmapfilenoACCESS_READr   r   )selffmms      [/var/www/auris/envauris/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.pyas_mmapDDUFEntry.as_mmap9   sr      ^^  &!188:a8H8HIRt{{T[['@AA J '&II '&s4   B,3B(B
9B	B,

B	B
B)%B,encodingc                     U R                   R                  S5       nUR                  U R                  5        UR	                  U R
                  5      R                  US9sSSS5        $ ! , (       d  f       g= f)zRead the file as text.

Useful for '.txt' and '.json' entries.

Example:
    ```py
    >>> import json
    >>> index = json.loads(entry.read_text())
    ```
r#   )r/   N)r    r%   seekr   readr   decode)r)   r/   r*   s      r,   	read_textDDUFEntry.read_textJ   sR     ^^  &!FF4;;66$++&--x-@ '&&s   AA))
A7 )zutf-8)__name__
__module____qualname____firstlineno____doc__str__annotations__intr   r    r   r   r	   bytesr-   r4   __static_attributes__r6       r,   r   r   "   sk     MKK'It'B5$#45 B B A# AC A ArA   r   r    r!   c           	      b   0 n[        U 5      n [        R                  SU  35        [        R                  " [        U 5      S5       nUR                  5        H  n[        R                  SUR                   35        UR                  [        R                  :w  a  [        S5      e [        UR                  5        [        X#5      n[        UR                  XSR                   U S9XR                  '   M     SSS5        SU;  a  [        S	5      e["        R$                  " US   R'                  5       5      n[)        XaR+                  5       5        [        R                  S
U  S[-        U5       S35        U$ ! [         a  n[        SUR                   35      UeSnAff = f! , (       d  f       N= f)a  
Read a DDUF file and return a dictionary of entries.

Only the metadata is read, the data is not loaded in memory.

Args:
    dduf_path (`str` or `os.PathLike`):
        The path to the DDUF file to read.

Returns:
    `Dict[str, DDUFEntry]`:
        A dictionary of [`DDUFEntry`] indexed by filename.

Raises:
    - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).

Example:
    ```python
    >>> import json
    >>> import safetensors.torch
    >>> from huggingface_hub import read_dduf_file

    # Read DDUF metadata
    >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf")

    # Returns a mapping filename <> DDUFEntry
    >>> dduf_entries["model_index.json"]
    DDUFEntry(filename='model_index.json', offset=66, length=587)

    # Load model index as JSON
    >>> json.loads(dduf_entries["model_index.json"].read_text())
    {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ...

    # Load VAE weights using safetensors
    >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm:
    ...     state_dict = safetensors.torch.load(mm)
    ```
zReading DDUF file rzReading entry z)Data must not be compressed in DDUF file.z!Invalid entry name in DDUF file: N)r   r   r   r    model_index.json7Missing required 'model_index.json' entry in DDUF file.zDone reading DDUF file z. Found z entries)r   loggerinfozipfileZipFiler<   infolistdebugr   compress_type
ZIP_STOREDr   _validate_dduf_entry_namer   _get_data_offsetr   	file_sizejsonloadsr4   _validate_dduf_structurekeyslen)r    entrieszfrG   er   indexs          r,   read_dduf_filerZ   Z   sg   N GYI
KK$YK01	Y	-KKMDLL>$--9:!!W%7%77,-XYYi)$--8 &b/F%.vnnXa&GMM" " 
.$ ($%^__JJw12<<>?EULLN3
KK))HS\N(STN! - i,/PQUQ^Q^P_-`aghhi 
.	-s1   AF &E5;:F 5
F?FFF  
F.rV   c                 :   [         R                  SU  S35        [        5       nSn[        R                  " [        U 5      S[        R                  5       nU H  u  pVXR;   a  [        SU 35      eUR                  U5        US:X  a.   [        R                  " [        U5      R                  5       5      n [        U5      n[         R!                  S	U S
35        [#        XEU5        M     SSS5        Uc  [        S5      e [%        X25        [         R                  SU  35        g! [        R                   a  n[        S5      UeSnAff = f! [         a  n[        SU 35      UeSnAff = f! , (       d  f       N= f! [&         a  n[        S5      UeSnAff = f)a  Write a DDUF file from an iterable of entries.

This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data.
In particular, you don't need to save the data on disk before exporting it in the DDUF file.

Args:
    dduf_path (`str` or `os.PathLike`):
        The path to the DDUF file to write.
    entries (`Iterable[Tuple[str, Union[str, Path, bytes]]]`):
        An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content.
        The filename should be the path to the file in the DDUF archive.
        The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes.

Raises:
    - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.).

Example:
    ```python
    # Export specific files from the local disk.
    >>> from huggingface_hub import export_entries_as_dduf
    >>> export_entries_as_dduf(
    ...     dduf_path="stable-diffusion-v1-4-FP16.dduf",
    ...     entries=[ # List entries to add to the DDUF file (here, only FP16 weights)
    ...         ("model_index.json", "path/to/model_index.json"),
    ...         ("vae/config.json", "path/to/vae/config.json"),
    ...         ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"),
    ...         ("text_encoder/config.json", "path/to/text_encoder/config.json"),
    ...         ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"),
    ...         # ... add more entries here
    ...     ]
    ... )
    ```

    ```python
    # Export state_dicts one by one from a loaded pipeline
    >>> from diffusers import DiffusionPipeline
    >>> from typing import Generator, Tuple
    >>> import safetensors.torch
    >>> from huggingface_hub import export_entries_as_dduf
    >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
    ... # ... do some work with the pipeline

    >>> def as_entries(pipe: DiffusionPipeline) -> Generator[Tuple[str, bytes], None, None]:
    ...     # Build an generator that yields the entries to add to the DDUF file.
    ...     # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file.
    ...     # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time)
    ...     yield "vae/config.json", pipe.vae.to_json_string().encode()
    ...     yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict())
    ...     yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode()
    ...     yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict())
    ...     # ... add more entries here

    >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe))
    ```
zExporting DDUF file ''NwzCan't add duplicate entry: rD   z#Failed to parse 'model_index.json'.zInvalid entry name: zAdding entry 'z' to DDUF filerE   zInvalid DDUF file structure.zDone writing DDUF file )rF   rG   setrH   rI   r<   rM   r   addrQ   rR   _load_contentr3   JSONDecodeErrorrN   r   rK   _dump_content_in_archiverS   r   )r    rV   	filenamesrY   archiver   contentrX   s           r,   export_entries_as_ddufrf      s   t KK'	{!45IE	Yg.@.@	AW!(H$%(CH:&NOOMM(#--X JJ}W'='D'D'FGEP4X> LL>(>BC$W@! ") 
B( }WXXE 2 KK))56% ++ X)*OPVWWX
 - P%(<XJ&GHaOP 
B	A0 " E<=1DEsf   2E.-D%6E)E.E? %E
9EE

E.
E+E&&E++E..
E<?
F	FFfolder_pathc                    ^ [        T5      mS[        [        [        [         4      4U4S jjn[	        X" 5       5        g)a  
Export a folder as a DDUF file.

AUses [`export_entries_as_dduf`] under the hood.

Args:
    dduf_path (`str` or `os.PathLike`):
        The path to the DDUF file to write.
    folder_path (`str` or `os.PathLike`):
        The path to the folder containing the diffusion model.

Example:
    ```python
    >>> from huggingface_hub import export_folder_as_dduf
    >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev")
    ```
r!   c               3     >#    [        T5      R                  S5       H  n U R                  5       (       d  M  U R                  [        ;  a  [
        R                  SU  S35        MI  U R                  T5      n[        UR                  5      S:  a  [
        R                  SU  S35        M  UR                  5       U 4v   M     g 7f)Nz**/*zSkipping file 'z' (file type not allowed)   z"' (nested directories not allowed))r   globis_filesuffixDDUF_ALLOWED_ENTRIESrF   rK   relative_torU   partsas_posix)pathpath_in_archiverg   s     r,   _iterate_over_folder3export_folder_as_dduf.<locals>._iterate_over_folder  s     %**62D<<>>{{"66tf4MNO"..{;O?(()Q.tf4VWX!**,d22 3s   CCN)r   r
   r   r<   rf   )r    rg   rt   s    ` r,   export_folder_as_ddufrv      s9    $ {#K3(5d+;"< 3 9&:&<=rA   rd   r   re   c                    U R                  USSS9 n[        U[        [        45      (       a=  [        U5      nUR                  S5       n[        R
                  " XSS5        S S S 5        O6[        U[        5      (       a  UR                  U5        O[        SU S35      eS S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr]   T)force_zip64r#   i   zInvalid content type for z. Must be str, Path or bytes.)	r%   
isinstancer<   r   shutilcopyfileobjr?   writer   )rd   r   re   
archive_fhcontent_path
content_fhs         r,   rb   rb     s    	h	6*gT{++=L""4(J"":?K )(''W%!$=hZGd"eff 
7	6 )( 
7	6s#   8B;
B*"?B;*
B8	4B;;
C	c                     [        U [        [        45      (       a  [        U 5      R                  5       $ [        U [        5      (       a  U $ [        S[        U 5       S35      e)zgLoad the content of an entry as bytes.

Used only for small checks (not to dump content into archive).
z6Invalid content type. Must be str, Path or bytes. Got .)ry   r<   r   
read_bytesr?   r   type)re   s    r,   r`   r`   *  sX    
 'C;''G}''))	GU	#	# VW[\cWdVeefghhrA   
entry_namec                     SU R                  S5      S   -   [        ;  a  [        SU  35      eSU ;   a  [        SU  S35      eU R                  S5      n U R	                  S5      S:  a  [        SU  S35      eU $ )	Nr   zFile type not allowed: \z0Entry names must use UNIX separators ('/'). Got /   z-DDUF only supports 1 level of directory. Got )splitrn   r   stripcount)r   s    r,   rN   rN   7  s    
Zc"2&&.BB'*A*(NOOz'*Z[eZffg(hii!!#&Jq '*WXbWccd(effrA   rY   entry_namesc                 j  ^^ [        U [        5      (       d  [        S[        U 5       S35      eT Vs1 s H  nSU;   d  M  UR	                  S5      S   iM!     nnU HM  mTU ;  a  [        ST S35      e[        UU4S j[         5       5      (       a  M9  [        ST S	[         S35      e   g
s  snf )ai  
Consistency checks on the DDUF file structure.

Rules:
- The 'model_index.json' entry is required and must contain a dictionary.
- Each folder name must correspond to an entry in 'model_index.json'.
- Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json').

Args:
    index (Any):
        The content of the 'model_index.json' entry.
    entry_names (Iterable[str]):
        The list of entry names in the DDUF file.

Raises:
    - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
z>Invalid 'model_index.json' content. Must be a dictionary. Got r   r   r   zMissing required entry 'z' in 'model_index.json'.c              3   8   >#    U  H  nT S U 3T;   v   M     g7f)r   Nr6   ).0required_entryr   folders     r,   	<genexpr>+_validate_dduf_structure.<locals>.<genexpr>[  s$     rUq>fXQ~./;>Uqs   z!Missing required file in folder 'z!'. Must contains at least one of N)ry   dictr   r   r   anyDDUF_FOLDER_REQUIRED_ENTRIES)rY   r   entrydduf_foldersr   s    `  @r,   rS   rS   B  s    $ eT""$'efjkpfqerrs%tuu5@Q[EC5L'EKK$Q'[LQ(+CF8Kc)deerUqrrr(3F8;\]y\zz{| 	  Rs   
B0B0rW   rG   c                 h   U R                   c  [        S5      eUR                  nU R                   R                  U5        U R                   R	                  S5      n[        U5      S:  a  [        S5      e[        R                  USS S5      n[        R                  USS S5      nUS-   U-   U-   nU$ )a  
Calculate the data offset for a file in a ZIP archive.

Args:
    zf (`zipfile.ZipFile`):
        The opened ZIP file. Must be opened in read mode.
    info (`zipfile.ZipInfo`):
        The file info.

Returns:
    int: The offset of the file data in the ZIP archive.
z+ZipFile object must be opened in read mode.   zIncomplete local file header.      little)fpr   header_offsetr1   r2   rU   r>   
from_bytes)rW   rG   r   local_file_headerfilename_lenextra_field_lendata_offsets          r,   rO   rO   a  s     
uu}$%RSS &&M EEJJ}

2
"$%DEE >>"3Br":HELnn%6r"%=xHO  "$|3oEKrA   ),rQ   loggingr&   osrz   rH   
contextlibr   dataclassesr   r   pathlibr   typingr   r   r	   r
   r   r   errorsr   r   r   	getLoggerr7   rF   rn   r   r   PathLiker<   rZ   r?   rf   rv   rI   rb   r`   rN   rS   ZipInfor>   rO   r6   rA   r,   <module>r      s      	   % (  ? ? W W 
		8	$    4A 4A 4AnBeBKK$45 B$sI~:N BJX7S"++%&X719%U3PTV[K[E\@\:]1^X7	X7v!>U3+;%< !>5QTVXVaVaQaKb !>gk !>H	ggoo 	g 	guUXZ\ZeZeglUlOm 	grv 	g
i5dE!12 
iu 
i# # C hsm  >" " "C "rA   