o Zh@s8UdZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl ZddlmZmZmZddlmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(e%rlddl)Z)e&*e+Z,eZ-eZ.gd Z/gd Z0d Z1e2e3d<GdddZ4e(ddGdddeZ5dgZ6dS)z Image processor class for Flava.N) lru_cache)AnyDictIterableListOptionalTupleUnion)BaseImageProcessor BatchFeature get_size_dict)resizeto_channel_dimension_format) OPENAI_CLIP_MEANOPENAI_CLIP_STDChannelDimension ImageInputPILImageResamplinginfer_channel_dimension_formatis_scaled_imagemake_list_of_imagesto_numpy_array valid_imagesvalidate_preprocess_arguments) TensorTypefilter_out_non_signature_kwargsis_vision_availablelogging)requires)r r )?r!r!g?LOGIT_LAPLACE_EPSc@sxeZdZ      ddeeeeeffdedeed ed eed eef d d ZddZ ddZ ddZ ddZ dS)FlavaMaskingGeneratorKN333333? input_sizetotal_mask_patchesmask_group_max_patchesmask_group_min_patchesmask_group_min_aspect_ratiomask_group_max_aspect_ratiocCsrt|ts |fd}|\|_|_|j|j|_||_||_|dur#|n||_|p+d|}t |t |f|_ dS)N) isinstancetupleheightwidthZ num_patchesr)r+r*mathloglog_aspect_ratio)selfr(r)r*r+r,r-r8_/var/www/auris/lib/python3.10/site-packages/transformers/models/flava/image_processing_flava.py__init__<s   zFlavaMaskingGenerator.__init__c Cs0d|j|j|j|j|j|jd|jdf}|S)Nz [%d ~ %d], max = %d, %.3f ~ %.3f)rr/)r2r3r+r*r)r6)r7repr_strr8r8r9__repr__Rs zFlavaMaskingGenerator.__repr__cCs |j|jfS)Nr2r3)r7r8r8r9 get_shape^s zFlavaMaskingGenerator.get_shapecCs4d}tdD]}t|j|}ttj|j}ttt ||}ttt ||}||j kr||j krt d|j |} t d|j |} || | || | |f } d||| krh|krnn&t| | |D]} t| | |D]} || | fdkrd|| | f<|d7}qzqq|dkr|Sq|S)Nr r/)rangerandomuniformr+r4expr6introundsqrtr3r2randintsum)r7maskmax_mask_patchesdeltaZ_attemptZ target_areaZ aspect_ratior2r3topleftZ num_maskedijr8r8r9_maskas.    zFlavaMaskingGenerator._maskcCsftj|td}d}||jkr1|j|}t||j}|||}|dkr( |S||7}||jks|S)N)shapeZdtyper)npZzerosr>rDr)minr*rP)r7rIZ mask_countrJrKr8r8r9__call__ys      zFlavaMaskingGenerator.__call__)r$r%Nr&r'N) __name__ __module__ __qualname__r rDrrfloatr:r<r>rPrTr8r8r8r9r#;s0   r#)Zvision)backendscGsVeZdZdZdgZddejddddddddddd dd ddddejddddddddfd ed e e e e fd edede e e e fdede e efdede e eeefde e eeefdede de de de e dede ededede ede d ed!e e d"ed#e e efd$ed%ed&e e eeefd'e e eeefd(df<fd)d* Zed+e e efffd,d- Zed(efd.d/Zejddfd0ejd e e e fd ed1e e e efd2e e e efd(ejf d3d4Zd0ejd(ejfd5d6Zdddddddddddejdf d0ed e ed e e e e fd ede ede e e e fde ede ede ede e eeefde e eeefd7e ed1e ed2e ed(ejfd8d9Ze ddddddddddddddddddddddddddddddejdf d:ed e ed e e e e fd ede ede e e e fde ede ede ede e eeefde e eeefde ede e de e de e de e de ede ede ede ede e e e fde e d e ed!e e e e fd"e ed#e ed$e ed%e ed&e eed'e eed;e e e e!fd1ed2e e e efd(e"j#j#fDdFlavaImageProcessora  Constructs a Flava image processor. Args: do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in `preprocess`. size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`): Size of the image after resizing. Can be overridden by the `size` parameter in `preprocess`. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use if resizing the image. Can be overridden by the `resample` parameter in `preprocess`. do_center_crop (`bool`, *optional*, defaults to `True`): Whether to center crop the images. Can be overridden by the `do_center_crop` parameter in `preprocess`. crop_size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`): Size of image after the center crop `(crop_size["height"], crop_size["width"])`. Can be overridden by the `crop_size` parameter in `preprocess`. do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the `do_rescale` parameter in `preprocess`. rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in `preprocess`. do_normalize (`bool`, *optional*, defaults to `True`): Whether to normalize the image. Can be overridden by the `do_normalize` parameter in `preprocess`. image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): Standard deviation to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. return_image_mask (`bool`, *optional*, defaults to `False`): Whether to return the image mask. Can be overridden by the `return_image_mask` parameter in `preprocess`. input_size_patches (`int`, *optional*, defaults to 14): Number of patches in the image in height and width direction. 14x14 = 196 total patches. Can be overridden by the `input_size_patches` parameter in `preprocess`. total_mask_patches (`int`, *optional*, defaults to 75): Total number of patches that should be masked. Can be overridden by the `total_mask_patches` parameter in `preprocess`. mask_group_min_patches (`int`, *optional*, defaults to 16): Minimum number of patches that should be masked. Can be overridden by the `mask_group_min_patches` parameter in `preprocess`. mask_group_max_patches (`int`, *optional*): Maximum number of patches that should be masked. Can be overridden by the `mask_group_max_patches` parameter in `preprocess`. mask_group_min_aspect_ratio (`float`, *optional*, defaults to 0.3): Minimum aspect ratio of the mask window. Can be overridden by the `mask_group_min_aspect_ratio` parameter in `preprocess`. mask_group_max_aspect_ratio (`float`, *optional*): Maximum aspect ratio of the mask window. Can be overridden by the `mask_group_max_aspect_ratio` parameter in `preprocess`. codebook_do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the input for codebook to a certain. Can be overridden by the `codebook_do_resize` parameter in `preprocess`. `codebook_size`. codebook_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): Resize the input for codebook to the given size. Can be overridden by the `codebook_size` parameter in `preprocess`. codebook_resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.LANCZOS`): Resampling filter to use if resizing the codebook image. Can be overridden by the `codebook_resample` parameter in `preprocess`. codebook_do_center_crop (`bool`, *optional*, defaults to `True`): Whether to crop the input for codebook at the center. If the input size is smaller than `codebook_crop_size` along any edge, the image is padded with 0's and then center cropped. Can be overridden by the `codebook_do_center_crop` parameter in `preprocess`. codebook_crop_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): Desired output size for codebook input when applying center-cropping. Can be overridden by the `codebook_crop_size` parameter in `preprocess`. codebook_do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the input for codebook by the specified scale `codebook_rescale_factor`. Can be overridden by the `codebook_do_rescale` parameter in `preprocess`. codebook_rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): Defines the scale factor to use if rescaling the codebook image. Can be overridden by the `codebook_rescale_factor` parameter in `preprocess`. codebook_do_map_pixels (`bool`, *optional*, defaults to `True`): Whether to map the pixel values of the codebook input to (1 - 2e)x + e. Can be overridden by the `codebook_do_map_pixels` parameter in `preprocess`. codebook_do_normalize (`bool`, *optional*, defaults to `True`): Whether or not to normalize the input for codebook with `codebook_image_mean` and `codebook_image_std`. Can be overridden by the `codebook_do_normalize` parameter in `preprocess`. codebook_image_mean (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0, 0, 0]`): The sequence of means for each channel, to be used when normalizing images for codebook. Can be overridden by the `codebook_image_mean` parameter in `preprocess`. codebook_image_std (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0.5, 0.5, 0.5]`): The sequence of standard deviations for each channel, to be used when normalizing images for codebook. Can be overridden by the `codebook_image_std` parameter in `preprocess`. pixel_valuesTNgp?Fr$r%r&r' do_resizesizeresampledo_center_crop crop_size do_rescalerescale_factor do_normalize image_mean image_stdreturn_image_maskinput_size_patchesr)r+r*r,r-return_codebook_pixelscodebook_do_resize codebook_sizecodebook_resamplecodebook_do_center_cropcodebook_crop_sizecodebook_do_rescalecodebook_rescale_factorcodebook_do_map_pixelscodebook_do_normalizecodebook_image_meancodebook_image_stdreturnc stjdi||dur|nddd}t|}|dur|nddd}t|dd}|dur/|nddd}t|dd}|dur@|nddd}t|dd}||_||_||_||_||_||_||_ ||_ | duri| nt |_ | durr| nt |_| |_| |_| |_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_ |dur|nt!|_ |dur||_#dSt"|_#dS) Nr=r` param_nameprjrmr8)$superr:r r\r]r^rarbr_r`rcFLAVA_IMAGE_MEANrdFLAVA_IMAGE_STDrerfrgr)r+r*r,r-rhrirjrkrlrmrnrorprqrrFLAVA_CODEBOOK_MEANFLAVA_CODEBOOK_STDrs)r7r\r]r^r_r`rarbrcrdrerfrgr)r+r*r,r-rhrirjrkrlrmrnrorprqrrrskwargs __class__r8r9r:sN#   zFlavaImageProcessor.__init__image_processor_dictc sH|}d|vr|d|d<d|vr|d|d<tj|fi|S)z Overrides the `from_dict` method from the base class to make sure parameters are updated if image processor is created using from_dict and kwargs e.g. `FlavaImageProcessor.from_pretrained(checkpoint, codebook_size=600)` rjrm)copypopry from_dict)clsrr~rr8r9r3s zFlavaImageProcessor.from_dictcCst||||||dS)N)r(r)r+r*r,r-)r#)r7rgr)r+r*r,r-r8r8r9masking_generator@s z%FlavaImageProcessor.masking_generatorimage data_formatinput_data_formatcKsTt|}d|vs d|vrtd||d|df}t|f||||d|S)a Resize an image to `(size["height"], size["width"])`. Args: image (`np.ndarray`): Image to resize. size (`Dict[str, int]`): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the output image. If unset, the channel dimension format of the input image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the input image. If unset, the channel dimension format is inferred from the input image. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. r2r3zFThe `size` dictionary must contain the keys `height` and `width`. Got )r]r^rr)r ValueErrorkeysr)r7rr]r^rrr~Z output_sizer8r8r9rTs#zFlavaImageProcessor.resizecCsddt|tS)Nr/r.)r")r7rr8r8r9 map_pixelsszFlavaImageProcessor.map_pixels do_map_pixelsc Cst||| | | |||||d t|}|rt|rtd|dur%t|}|r0|j||||d}|r:|j|||d}|rD|j|||d}| rO|j || | |d}| rV| |}| durat || |d}|S) zPreprocesses a single image.) rarbrcrdrer_r`r\r]r^zIt looks like you are trying to rescale already rescaled images. If the input images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again.N)rr]r^r)rr]r)rscaler)rmeanZstdr)Zinput_channel_dim) rrrloggerZ warning_oncerrZ center_cropZrescale normalizerr)r7rr\r]r^r_r`rarbrcrdrerrrr8r8r9_preprocess_images@  z%FlavaImageProcessor._preprocess_imageimagesreturn_tensorsc"&sdurnjdurnjtdurnj dur% nj dur. nj t dd dur=njdurFnjdurOnjdurXnj duranj | durj| nj } | durs| nj } |dur||nj }|dur|nj}|dur|nj}|dur|nj}|dur|nj}|dur|nj}durnj dur nj t dd durʈnjdurӈnj dur܈ nj durnjdurnjtdddurnjdurnjdurnjdurnjt|}t|s+t d f dd|D}"d |"i}#|ra fd d|D}$|$|#d <| r|j!| |||||d fd d|D}%|%|#d<t"|#|dS)a Preprocess an image or batch of images. Args: images (`ImageInput`): Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. do_resize (`bool`, *optional*, defaults to `self.do_resize`): Whether to resize the image. size (`Dict[str, int]`, *optional*, defaults to `self.size`): Size of the image. resample (`int`, *optional*, defaults to `self.resample`): Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`, Only has an effect if `do_resize` is set to `True`. do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): Whether to center crop the image. crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): Size of the center crop. Only has an effect if `do_center_crop` is set to `True`. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): Whether to rescale the image values between [0 - 1]. rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): Rescale factor to rescale the image by if `do_rescale` is set to `True`. do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): Whether to normalize the image. image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): Image mean. image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): Image standard deviation. return_image_mask (`bool`, *optional*, defaults to `self.return_image_mask`): Whether to return the image mask. input_size_patches (`int`, *optional*, defaults to `self.input_size_patches`): Size of the patches to extract from the image. total_mask_patches (`int`, *optional*, defaults to `self.total_mask_patches`): Total number of patches to extract from the image. mask_group_min_patches (`int`, *optional*, defaults to `self.mask_group_min_patches`): Minimum number of patches to extract from the image. mask_group_max_patches (`int`, *optional*, defaults to `self.mask_group_max_patches`): Maximum number of patches to extract from the image. mask_group_min_aspect_ratio (`float`, *optional*, defaults to `self.mask_group_min_aspect_ratio`): Minimum aspect ratio of the patches to extract from the image. mask_group_max_aspect_ratio (`float`, *optional*, defaults to `self.mask_group_max_aspect_ratio`): Maximum aspect ratio of the patches to extract from the image. return_codebook_pixels (`bool`, *optional*, defaults to `self.return_codebook_pixels`): Whether to return the codebook pixels. codebook_do_resize (`bool`, *optional*, defaults to `self.codebook_do_resize`): Whether to resize the codebook pixels. codebook_size (`Dict[str, int]`, *optional*, defaults to `self.codebook_size`): Size of the codebook pixels. codebook_resample (`int`, *optional*, defaults to `self.codebook_resample`): Resampling filter to use if resizing the codebook pixels. This can be one of the enum `PILImageResampling`, Only has an effect if `codebook_do_resize` is set to `True`. codebook_do_center_crop (`bool`, *optional*, defaults to `self.codebook_do_center_crop`): Whether to center crop the codebook pixels. codebook_crop_size (`Dict[str, int]`, *optional*, defaults to `self.codebook_crop_size`): Size of the center crop of the codebook pixels. Only has an effect if `codebook_do_center_crop` is set to `True`. codebook_do_rescale (`bool`, *optional*, defaults to `self.codebook_do_rescale`): Whether to rescale the codebook pixels values between [0 - 1]. codebook_rescale_factor (`float`, *optional*, defaults to `self.codebook_rescale_factor`): Rescale factor to rescale the codebook pixels by if `codebook_do_rescale` is set to `True`. codebook_do_map_pixels (`bool`, *optional*, defaults to `self.codebook_do_map_pixels`): Whether to map the codebook pixels values. codebook_do_normalize (`bool`, *optional*, defaults to `self.codebook_do_normalize`): Whether to normalize the codebook pixels. codebook_image_mean (`float` or `List[float]`, *optional*, defaults to `self.codebook_image_mean`): Codebook pixels mean to normalize the codebook pixels by if `codebook_do_normalize` is set to `True`. codebook_image_std (`float` or `List[float]`, *optional*, defaults to `self.codebook_image_std`): Codebook pixels standard deviation to normalize the codebook pixels by if `codebook_do_normalize` is set to `True`. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: - Unset: Return a list of `np.ndarray`. - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format for the input image. If unset, the channel dimension format is inferred from the input image. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Nr`rvrjrmzkInvalid image type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.cs2g|]} j|  ddqS)Frr\r]r^r_r`rarbrcrdrerrrr.0Zimg) r`rr_rcrar\rdrerr^rbr7r]r8r9 &z2FlavaImageProcessor.preprocess..r[cs2g|]} j|   dqS)rrr)rmrlrprqrnrirrrsrkrorjrrr7r8r9rrZcodebook_pixel_values)rgr)r+r*r,r-csg|]}qSr8r8)r_)mask_generatorr8r9rsZbool_masked_pos)dataZ tensor_type)#r\r]r r^r_r`rarbrcrdrerfrgr)r+r*r,r-rhrirjrkrnrorlrmrprqrrrsrrrrr )&r7rr\r]r^r_r`rarbrcrdrerfrgr)r+r*r,r-rhrirjrkrlrmrnrorprqrrrsrrrZprocessed_imagesrZcodebook_imagesmasksr8)rmrlrprqrnrirrrsrkrorjr`rr_rcrar\rdrerrr^rbr7r]r9 preprocesss~    "$ zFlavaImageProcessor.preprocess)&rUrVrW__doc__Zmodel_input_namesrZBICUBICZLANCZOSboolrrstrrDr rXrr: classmethodrrrr#rrRZndarrayrrrZFIRSTrrrrrPILZImager __classcell__r8r8rr9rZs W       "O    0      @      !"#$%rZ)7rr4rA functoolsrtypingrrrrrrr numpyrRZimage_processing_utilsr r r Zimage_transformsrrZ image_utilsrrrrrrrrrrrutilsrrrrZutils.import_utilsrrZ get_loggerrUrrzr{r|r}r"rX__annotations__r#rZ__all__r8r8r8r9s6 $4   N ;