a h*C@sddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z ddl Z ddl mZmZmZmZddlmZe dZejfeeeeedd d Zde jeeee jd d d ZGdddZeedddZGdddZdS)NFraction)AnyCallablecastOptionalTypeVarUnion)_probe_video_from_file_read_video_from_file read_videoread_video_timestamps)tqdmT)pts timebase_from timebase_to round_funcreturncCst|d||}||S)zconvert pts between different time bases Args: pts: presentation timestamp, float timebase_from: original timebase. Fraction timebase_to: new timebase. Fraction round_func: rounding function. rr)rrrrZnew_ptsrN/var/www/auris/lib/python3.9/site-packages/torchvision/datasets/video_utils.py pts_convertsr)tensorsizestepdilationrcCs|dkrtd||d}|}||||f}|||dd|d|f}|ddkrtd|f}t|||S)a similar to tensor.unfold, but with the dilation and specialized for 1d tensors Returns all consecutive windows of `size` elements, with `step` between windows. The distance between each element in a window is given by `dilation`. rz*tensor should have 1 dimension instead of r)dim ValueErrorZstridenumeltorchZ as_strided)rrrrZo_striderZ new_strideZnew_sizerrrunfolds    r!c@sReZdZdZeeddddZedddZee eee e fd d d Z dS) _VideoTimestampsDatasetz Dataset used to parallelize the reading of the timestamps of a list of videos, given their paths in the filesystem. Used in VideoClips and defined at top level, so it can be pickled when forking. N) video_pathsrcCs ||_dSNr#)selfr#rrr__init__8sz _VideoTimestampsDataset.__init__rcCs t|jSr$lenr#r&rrr__len__;sz_VideoTimestampsDataset.__len__idxrcCst|j|Sr$)r r#)r&r.rrr __getitem__>sz#_VideoTimestampsDataset.__getitem__) __name__ __module__ __qualname____doc__liststrr'intr,tuplerfloatr/rrrrr"/sr")xrcCs|S)zH Dummy collate function to be used with _VideoTimestampsDataset r)r9rrr _collate_fnBsr:c@seZdZdZd.eeeeeeee ee feeeeeeeedddd Z dd d d Z e ee fdd ddZ ee ee fd ddZeeddddZed/ejeeeeeeeejeeeejffdddZd0eeeeddddZed ddZed ddZed dd Zeeeefd!d"d#Zeeeeeeejfd$d%d&Zeeejeje ee fefd!d'd(Ze ee fd d)d*Ze ee fdd+d,d-Z dS)1 VideoClipsaE Given a list of video files, computes all consecutive subvideos of size `clip_length_in_frames`, where the distance between each subvideo in the same video is defined by `frames_between_clips`. If `frame_rate` is specified, it will also resample all the videos to have the same frame rate, and the clips will refer to this frame rate. Creating this instance the first time is time-consuming, as it needs to decode all the videos in `video_paths`. It is recommended that you cache the results after instantiation of the class. Recreating the clips for different clip lengths is fast, and can be done with the `compute_clips` method. Args: video_paths (List[str]): paths to the video files clip_length_in_frames (int): size of a clip in number of frames frames_between_clips (int): step (in frames) between each clip frame_rate (float, optional): if specified, it will resample the video so that it has `frame_rate`, and then the clips will be defined on the resampled video num_workers (int): how many subprocesses to use for data loading. 0 means that the data will be loaded in the main process. (default: 0) output_format (str): The format of the output video tensors. Can be either "THWC" (default) or "TCHW". rNrTHWC)r#clip_length_in_framesframes_between_clips frame_rate_precomputed_metadata num_workers _video_width _video_height_video_min_dimension_video_max_dimension_audio_samples_audio_channels output_formatrcCs||_||_||_||_| |_| |_| |_| |_| |_ |j dvrTt d| d|durf| n | || |||dS)N)r=TCHWz5output_format should be either 'THWC' or 'TCHW', got .)r#rBrCrDrErFrGrHupperrIr_compute_frame_pts_init_from_metadata compute_clips)r&r#r>r?r@rArBrCrDrErFrGrHrIrrrr'ds    zVideoClips.__init__r(csg|_g|_ddljjjt|jd|jt d}t t |d^}|D]H}| dt t|\}}fdd|D}|j||j|qFWdn1s0YdS)Nrr<)Z batch_sizerBZ collate_fn)totalrcsg|]}j|jdqS)Zdtype) as_tensorlong).0rr rr z1VideoClips._compute_frame_pts..) video_pts video_fpsZtorch.utils.datautilsdataZ DataLoaderr"r#rBr:rr*updater4zipextend)r&dlZpbarbatchZ batch_ptsZ batch_fpsrrUrrMs   zVideoClips._compute_frame_pts)metadatarcCsV|d|_t|jt|dks$J|d|_t|jt|dksHJ|d|_dS)Nr#rXrY)r#r*rXrY)r&rarrrrNs   zVideoClips._init_from_metadatacCs|j|j|jd}|S)Nr#rXrYrb)r& _metadatarrrras zVideoClips.metadata)indicesrcs~fdd|D}fdd|D}fdd|D}|||d}t|jjj|jjjjjj j j d S)Ncsg|]}j|qSrr%rTir+rrrVrWz%VideoClips.subset..csg|]}j|qSr)rXrer+rrrVrWcsg|]}j|qSr)rYrer+rrrVrWrb) r>r?r@rArBrCrDrErFrGrHrI) type num_framesrr@rBrCrDrErFrGrHrI)r&rdr#rXrYrarr+rsubsets,zVideoClips.subset)rXrhrfpsr@rc Cs|dur d}|dur|}t|||}ttt|||}||}t|||}|sft dt |t r|gt|}n t|||}||fS)NrzThere aren't enough frames in the current video to get a clip for the given clip length and frames between clips. The video (and potentially others) will be skipped.) r*r;_resample_video_idxr6mathfloorr!rwarningswarn isinstanceslice) rXrhrrjr@Z total_framesZ_idxsclipsidxsrrrcompute_clips_for_videos    z"VideoClips.compute_clips_for_video)rhrr@rc Cs||_||_||_g|_g|_t|j|jD]6\}}||||||\}}|j ||j |q,t dd|jD}| d |_dS)a Compute all consecutive sequences of clips from video_pts. Always returns clips of size `num_frames`, meaning that the last few frames in a video can potentially be dropped. Args: num_frames (int): number of frames for the clip step (int): distance between two clips frame_rate (int, optional): The frame rate cSsg|] }t|qSrr*rTvrrrrVrWz,VideoClips.compute_clips..rN)rhrr@rrresampling_idxsr]rXrYrtappendr rRZcumsumtolistcumulative_sizes) r&rhrr@rXrjrrrsZ clip_lengthsrrrrOs  zVideoClips.compute_clipscCs|Sr$) num_clipsr+rrrr,szVideoClips.__len__cCs t|jSr$r)r+rrr num_videosszVideoClips.num_videoscCs |jdS)zJ Number of subclips that are available in the video list. )r{r+rrrr|szVideoClips.num_clipsr-cCs6t|j|}|dkr|}n||j|d}||fS)zw Converts a flattened representation of the indices into a video_idx, clip_idx representation. rr)bisect bisect_rightr{)r&r. video_idxclip_idxrrrget_clip_location s zVideoClips.get_clip_location)rh original_fpsnew_fpsrcCsL||}|r$t|}tdd|Stj|tjd|}|tj}|S)NrQ) is_integerr6rqr Zarangefloat32rmtoint64)rhrrrrsrrrrks zVideoClips._resample_video_idxc Cst||kr&td|d|d||\}}|j|}|j||}ddlm}|}|dkr|jdkrxtd|j dkrtd|j dkrtd |j dkrtd |j dkrtd |dkr|d }|d  } t||| \} } } nt|} | j}d }tt|d }tt|d  }d\}}tdd}t| jj| jj}| jrt| jj| jj}t|||tj}t|||tj}| j}t||j|j |j |j ||f||j |j||f|d \} } }d|i} |d ur|| d<|j d ur&|j!||}t"|t#j$r||d}| |} |j | d<t%| |j&ksLJ| j'd|j&|j(dkrh| )dddd} | | | |fS)a7 Gets a subclip from a list of videos. Args: idx (int): index of the subclip. Must be between 0 and num_clips(). Returns: video (Tensor) audio (Tensor) info (Dict) video_idx (int): index of the video in `video_paths` zIndex z out of range (z number of clips)r)get_video_backendZpyavz.pyav backend doesn't support _video_width != 0z/pyav backend doesn't support _video_height != 0z6pyav backend doesn't support _video_min_dimension != 0z6pyav backend doesn't support _video_max_dimension != 0z0pyav backend doesn't support _audio_samples != 0r~N)rr~r) Z video_widthZ video_heightZvideo_min_dimensionZvideo_max_dimensionZvideo_pts_rangevideo_timebaseZ audio_samplesZaudio_channelsZaudio_pts_rangeaudio_timebaserY audio_fpsz x rJ)*r| IndexErrorrr#rrZ torchvisionrrCrrDrErFrGitemr r rYrr6rr numerator denominatorZ has_audiorrrlrmceilZaudio_sample_rater rHr@rxrpr Tensorr*rhshaperIZpermute)r&r.rrZ video_pathZclip_ptsrbackendZ start_ptsZend_ptsvideoaudioinfo_inforYrZvideo_start_ptsZ video_end_ptsZaudio_start_ptsZ audio_end_ptsrr_Zresampling_idxrrrget_clip!sz                & zVideoClips.get_clipcCsndd|jD}dd|jD}|r6t|}|}|j}||d<||d<|d=|d=|d=d |d <|S) NcSsg|] }t|qSrrurvrrrrVzrWz+VideoClips.__getstate__..cSsg|]}|tjqSr)rr r)rTr9rrrrVrWvideo_pts_sizesrXrrrxr{r_version)rXr catnumpy__dict__copy)r&rrXdrrr __getstate__ys  zVideoClips.__getstate__)rrcCsfd|vr||_dStj|dtjd}tj||ddd}|d=||d<||_||j|j|jdS)NrrXrQrr)r) rr rRrsplitrOrhrr@)r&rrXrrr __setstate__szVideoClips.__setstate__) r<rNNrrrrrrrr=)N)N)!r0r1r2r3r4r5r6rr8dictrr'rMrNpropertyrari staticmethodr rr7r rqrtrOr,r}r|rrkrrrrrrrr;Isd %  (Xr;)r)rrlrnZ fractionsrtypingrrrrrr r Ztorchvision.ior r r r rZrrrmr6rrr!r"r:r;rrrrs