Welcome to Doom9's Forum, THE in-place to be for everyone interested in DVD conversion. Before you start posting please read the forum rules. By posting to this forum you agree to abide by the rules. |
![]() |
#1 | Link |
Registered User
Join Date: Oct 2001
Location: Germany
Posts: 7,566
|
Fill black frames with interpolations,...
Hi, I wrote two small functions:
Problem is FillBlackFrames, where I suspect that I messed up indices somewhere: Code:
# replace the frames 'firstFrametoReplace' to 'firstGoodFrame-1' with interpolations between 'firstFrametoReplace - 1' and 'firstGoodFrame' def fillMultipleWithSVP(clip, firstFrametoReplace=None, firstGoodFrame=None, gpu=False): clip1 = core.std.AssumeFPS(clip, fpsnum=1, fpsden=1) start = core.std.Trim(clip1, first=firstFrametoReplace-1, length=1) # last good frame before black frames end = core.std.Trim(clip1, first=firstGoodFrame, length=1) # first good frame after black frames startend = start + end if gpu: super = core.svp1.Super(startend,"{gpu:1}") else: super = core.svp1.Super(startend,"{gpu:0}") vectors= core.svp1.Analyse(super["clip"],super["data"],startend,"{}") # interpolate r = core.svp2.SmoothFps(startend,super["clip"],super["data"],vectors["clip"],vectors["data"],"{rate:{num:"+str(firstGoodFrame-firstFrametoReplace+1)+",den:1,abs:false}}") a = core.std.Trim(clip1, first=0, last=firstFrametoReplace-2) # last good frame before is part of r b = core.std.Trim(clip1, first=firstGoodFrame+1) # first good frame, ist part of r # join join = a + r + b if (join.num_frames != clip.num_frames): # did I messup with the join raise vs.Error(f"fillMultipleWithSVP: frame count issue join '{join.num_frames}' vs clip '{clip.num_frames}'") return core.std.AssumeFPS(join, src=clip) # replace a black frames with interpolations of the surrounding frames. # threshold average frame threshold def FillBlackFrames(clip, thresh=0.1, debug=False): if not isinstance(clip, vs.VideoNode): raise ValueError('This is not a clip') def selectFunc(n, f): nonlocal clip firstGoodFrame = f.props['FirstGoodFrame'] if n == 0: clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=firstGoodFrame) return clip if firstGoodFrame >= n: # this frame, we already dealt with return clip if f.props['PlaneStatsAverage'] > thresh or n == 0: if debug: return core.text.Text(clip=clip, text="Org, avg: " + str(f.props['PlaneStatsAverage']), alignment=8) clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=n) return clip firstFrametoReplace = n firstGoodFrame = n + 1 for i in range(n, clip.num_frames-1): if clip.get_frame(i).props['PlaneStatsAverage'] <= thresh: continue firstGoodFrame = i break clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=firstGoodFrame) replaced = fillMultipleWithSVP(clip, firstFrametoReplace, firstGoodFrame, gpu=True) if debug: return core.text.Text(clip=replaced, text="Replaced from " + str(firstFrametoReplace) + " to " + str(firstGoodFrame-1)) return replaced clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=0) clip = core.std.PlaneStats(clip) fixed = core.std.FrameEval(clip, selectFunc, prop_src=clip) return fixed When I have a clip where: Frame 19 is fine Frame 20&21 are black Frame 22 is fine and I apply FillBlackFrames to it, both frame 20&21 get replaced, but frame 21 is the same as 22. Does someone spot where I went wrong? (Or is this caused by Vapoursynth not really supporting linear processing in script?) Cu Selur Latest version:on GitHub Last edited by Selur; 31st May 2023 at 18:59. |
![]() |
![]() |
![]() |
#3 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
I just tried that first one, ReplaceBlackFrames, that you say it works, but anyway to simplify it because all it has to be done is to remember last non black n for a frame. To "remember" things is always a flag to introduce a class that just handles "globals" within itself.
ReplaceBlackFrames: Code:
import vapoursynth as vs from vapoursynth import core class RBF: def __init__(self, clip, thresh=0.1, debug=False): self.last_non_black_n = 0 self.clip = clip self.thresh = thresh self.debug = debug def selectFunc(self, n, f): if f.props['PlaneStatsAverage'] > self.thresh or n == 0: out = self.clip self.last_non_black_n = n else: out = self.clip[self.last_non_black_n] if self.debug: return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def ReplaceBlackFrames(clip, thresh=0.1, debug=False): clip = core.std.PlaneStats(clip) rbf = RBF(clip, thresh, debug) fixed = core.std.FrameEval(clip, rbf.selectFunc, prop_src=clip) return fixed clip1 = core.std.BlankClip(color=(255,0,0)) #red clip2 = core.std.BlankClip() #black clip3 = core.std.BlankClip(color=(255,255,255)) #white clip = clip1+clip2+clip3 clip = ReplaceBlackFrames(clip) clip.set_output() Last edited by _Al_; 29th May 2023 at 18:11. |
![]() |
![]() |
![]() |
#4 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
to simplify it even more, interestingly, this works too:
Code:
import vapoursynth as vs from vapoursynth import core class ReplaceBlackFrames: def __init__(self, clip, thresh=0.1, debug=False): self.last_non_black_n = 0 self.clip = clip.std.PlaneStats(clip) self.thresh = thresh self.debug = debug def selectFunc(self, n, f): if f.props['PlaneStatsAverage'] > self.thresh or n == 0: out = self.clip self.last_non_black_n = n else: out = self.clip[self.last_non_black_n] if self.debug: return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out @property def out(self): return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip) clip1 = core.std.BlankClip(color=(255,0,0)) #red clip2 = core.std.BlankClip() #black clip3 = core.std.BlankClip(color=(255,255,255)) #white clip = clip1+clip2+clip3 rbf = ReplaceBlackFrames(clip, debug=True) rbf.out.set_output() |
![]() |
![]() |
![]() |
#5 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
OK, :-), just tested it, it works in a linear frame request fashion, like enkoding etc., but in a previewer with random frame access it needs to be like Selur posted, hunting for a previous good frame in a loop (maybe there is another way, not sure):
Code:
import vapoursynth as vs from vapoursynth import core class ReplaceBlackFrames: def __init__(self, clip, thresh=0.1, debug=False): self.clip = clip.std.PlaneStats(clip) self.thresh = thresh self.debug = debug def selectFunc(self, n, f): out = self.clip[self.get_good_n(n)] if self.debug: return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def get_good_n(self, n): for i in reversed(range(n+1)): if self.clip.get_frame(i).props['PlaneStatsAverage'] > self.thresh: return i else: return n @property def out(self): return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip) clip1 = core.std.BlankClip(color=(255,0,0)) #red clip2 = core.std.BlankClip() #black clip3 = core.std.BlankClip(color=(255,255,255)) #white clip = clip1+clip2+clip3 rbf = ReplaceBlackFrames(clip, debug=True) rbf.out.set_output() Last edited by _Al_; 30th May 2023 at 02:34. |
![]() |
![]() |
![]() |
#6 | Link | |
Registered User
Join Date: Oct 2001
Location: Germany
Posts: 7,566
|
@_AI_:Thanks, will look at it later after work.
@kedautinh12: Quote:
|
|
![]() |
![]() |
![]() |
#7 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
As for FillBlackFrames, to interpolate black frames,
I came up with this, seams to work, but tested only with posted example, where I manually inserted 5 black frames into a video (made sure there was some movement) and it was successfully interpolated: Code:
import vapoursynth as vs from vapoursynth import core class FillBlackFrames: def __init__(self, clip, thresh=0.1, debug=False, gpu=False): self.clip = clip.std.PlaneStats(clip) self.thresh = thresh self.debug = debug self.gpu = gpu self.smooth = None def selectFunc(self, n, f): out = self.get_clip(n) if self.debug: return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def is_not_black(self, n): return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh def get_clip(self, n): if self.is_not_black(n): return self.clip[n] for start in reversed(range(n+1)): if self.is_not_black(start): break else: #there are all black frames preceding n, return current n frame return self.clip[n] for end in range(n, len(self.clip)): if self.is_not_black(end): break else: #there are all black frames to the end, return current n frame return self.clip[n] #does interpolated smooth clip exists for requested n frame? Use n frame from it. if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end: return self.smooth[n-start] #interpolating two frame clip into end-start+1 fps clip = self.clip[start] + self.clip[end] clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1) if self.gpu: super = core.svp1.Super(clip,"{gpu:1}") else: super = core.svp1.Super(clip,"{gpu:0}") vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}") num = end - start +1 self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}") self.smooth_start = start self.smooth_end = end return self.smooth[n-start] @property def out(self): return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip) clip1 = core.lsmas.LibavSMASHSource(r'F:\video.mp4')[500:530] clip2 = clip1.std.BlankClip()[:5] # inserted 5 black frames clip3 = core.lsmas.LibavSMASHSource(r'F:\video.mp4')[535:700] clip = clip1+clip2+clip3 fbf = FillBlackFrames(clip, debug=True) fbf.out.set_output() Does it interpolate better if it has more frames available on those sides? I guess yes, I might add it later , to use more frames if it is possible. Last edited by _Al_; 30th May 2023 at 07:07. |
![]() |
![]() |
![]() |
#8 | Link |
Registered User
Join Date: Oct 2001
Location: Germany
Posts: 7,566
|
Hmm,.. I changed it to:
Code:
import vapoursynth as vs from vapoursynth import core ''' call using: from ReplaceBlackFrames import ReplaceBlackFrames rbf = ReplaceBlackFrames(clip, debug=True, thresh=0.1, method='previous') clip = rbf.out debug: whether to display the average luma method: 'previous': replace black frames with the last non-black frame 'interpolateSVP': replace black frames whit interpolatied frames using SVP (GPU) 'interpolateSVPCPU': replace black frames whit interpolatied frames using SVP (CPU) ''' class ReplaceBlackFrames: # constructor def __init__(self, clip: vs.VideoNode, thresh: float=0.1, debug: bool=False, method: str='previous'): self.clip = clip.std.PlaneStats(clip) self.thresh = thresh self.debug = debug self.method = method def previous(self, n, f): if f.props['PlaneStatsAverage'] > self.thresh or n == 0: out = self.clip self.last_non_black_n = n else: out = self.clip[self.last_non_black_n] if self.debug: return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def interpolate(self, n, f): out = self.get_clip(n) if self.debug: return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def is_not_black(self, n): return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh def get_clip(self, n): if self.is_not_black(n): return self.clip[n] for start in reversed(range(n+1)): if self.is_not_black(start): break else: #there are all black frames preceding n, return current n frame return self.clip[n] for end in range(n, len(self.clip)): if self.is_not_black(end): break else: #there are all black frames to the end, return current n frame return self.clip[n] #does interpolated smooth clip exists for requested n frame? Use n frame from it. if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end: return self.smooth[n-start] #interpolating two frame clip into end-start+1 fps clip = self.clip[start] + self.clip[end] clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1) if self.method == 'interpolateSVP': super = core.svp1.Super(clip,"{gpu:1}") else: super = core.svp1.Super(clip,"{gpu:0}") vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}") num = end - start +1 self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}") self.smooth_start = start self.smooth_end = end return self.smooth[n-start] @property def out(self): if self.method == 'previous': return core.std.FrameEval(self.clip, self.previous, prop_src=self.clip) else: return core.std.FrameEval(self.clip, self.interpolate, prop_src=self.clip) Sadly, only the 'previous' (= old ReplaceBlackFrames) work. Using 'interpolateSVP' I end with the output not having interpolated frames. No clue where I broke it. ![]() I uploaded my test file and my test script to my GoogleDrive. Cu Selur |
![]() |
![]() |
![]() |
#9 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
In constructor method, __init__(), add
Code:
self.smooth=None Code:
for start in reversed(range(n+1)): if self.is_not_black(start): break else: #there are all black frames preceding n, return current n frame return self.clip[n] for end in range(n, len(self.clip)): if self.is_not_black(end): break else: #there are all black frames to the end, return current n frame return self.clip[n] Also for that simple previous good frame instead of black, you selected version that works the same only in linear fashion, I guess that second should be used. Last edited by _Al_; 31st May 2023 at 01:00. |
![]() |
![]() |
![]() |
#10 | Link |
Registered User
Join Date: May 2011
Posts: 364
|
I rather post your integrated solution, the whole thing, as it should be I guess, so there is no confusion:
Code:
import vapoursynth as vs from vapoursynth import core ''' call using: from ReplaceBlackFrames import ReplaceBlackFrames rbf = ReplaceBlackFrames(clip, debug=True, thresh=0.1, method='previous') clip = rbf.out debug: whether to display the average luma method: 'previous': replace black frames with the last non-black frame 'interpolateSVP': replace black frames whit interpolatied frames using SVP (GPU) 'interpolateSVPCPU': replace black frames whit interpolatied frames using SVP (CPU) ''' class ReplaceBlackFrames: # constructor def __init__(self, clip: vs.VideoNode, thresh: float=0.1, debug: bool=False, method: str='previous'): self.clip = core.std.PlaneStats(clip) self.thresh = thresh self.debug = debug self.method = method self.smooth = None def previous(self, n, f): out = self.get_current_or_previous(n) if self.debug: return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def interpolate(self, n, f): out = self.get_current_or_interpolate(n) if self.debug: return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8) return out def get_current_or_previous(self, n): for i in reversed(range(n+1)): if self.is_not_black(i): return self.clip[i] else: #all previous are black, return current n frame return self.clip[n] def get_current_or_interpolate(self, n): if self.is_not_black(n): #current non black selected return self.clip[n] #black frame, frame is interpolated for start in reversed(range(n+1)): if self.is_not_black(start): break else: #there are all black frames preceding n, return current n frame return self.clip[n] for end in range(n, len(self.clip)): if self.is_not_black(end): break else: #there are all black frames to the end, return current n frame return self.clip[n] #does interpolated smooth clip exist for requested n frame? Use n frame from it. if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end: return self.smooth[n-start] #interpolating two frame clip into end-start+1 fps clip = self.clip[start] + self.clip[end] clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1) if self.method == 'interpolateSVP': super = core.svp1.Super(clip,"{gpu:1}") else: super = core.svp1.Super(clip,"{gpu:0}") vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}") num = end - start + 1 self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}") self.smooth_start = start self.smooth_end = end return self.smooth[n-start] def is_not_black(self, n): return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh @property def out(self): if self.method == 'previous': return core.std.FrameEval(self.clip, self.previous, prop_src=self.clip) else: return core.std.FrameEval(self.clip, self.interpolate, prop_src=self.clip) Last edited by _Al_; 31st May 2023 at 02:08. |
![]() |
![]() |
![]() |
#11 | Link |
Registered User
Join Date: Oct 2001
Location: Germany
Posts: 7,566
|
Nice! Thanks a lot. This confused about how Python uses indention.
![]() ... Ahhh I got it, the 'else: ' part are executed if the for loop ends normally (not stopped by break) Cu Selur Ps.: but it on GitHub and added interpolation through RIFE to it. Last edited by Selur; 31st May 2023 at 19:00. |
![]() |
![]() |
![]() |
Thread Tools | Search this Thread |
Display Modes | |
|
|