Fill black frames with interpolations,...

[Selur]

Hi, I wrote two small functions:

ReplaceBlackFrames: replace a black frames with interpolations of the surrounding frames.
FillBlackFrames: replace a black frames with interpolations of the surrounding frames.

ReplaceBlackFrames works as expected.
Problem is FillBlackFrames, where I suspect that I messed up indices somewhere:
# replace the frames 'firstFrametoReplace' to 'firstGoodFrame-1' with interpolations between 'firstFrametoReplace - 1' and 'firstGoodFrame'
def fillMultipleWithSVP(clip, firstFrametoReplace=None, firstGoodFrame=None, gpu=False):
clip1 = core.std.AssumeFPS(clip, fpsnum=1, fpsden=1)
start = core.std.Trim(clip1, first=firstFrametoReplace-1, length=1) # last good frame before black frames
end = core.std.Trim(clip1, first=firstGoodFrame, length=1) # first good frame after black frames
startend = start + end
if gpu:
super = core.svp1.Super(startend,"{gpu:1}")
else:
super = core.svp1.Super(startend,"{gpu:0}")
vectors= core.svp1.Analyse(super["clip"],super["data"],startend,"{}")

# interpolate
r = core.svp2.SmoothFps(startend,super["clip"],super["data"],vectors["clip"],vectors["data"],"{rate:{num:"+str(firstGoodFrame-firstFrametoReplace+1)+",den:1,abs:false}}")
a = core.std.Trim(clip1, first=0, last=firstFrametoReplace-2) # last good frame before is part of r
b = core.std.Trim(clip1, first=firstGoodFrame+1) # first good frame, ist part of r
# join
join = a + r + b
if (join.num_frames != clip.num_frames): # did I messup with the join
raise vs.Error(f"fillMultipleWithSVP: frame count issue join '{join.num_frames}' vs clip '{clip.num_frames}'")
return core.std.AssumeFPS(join, src=clip)


# replace a black frames with interpolations of the surrounding frames.
# threshold average frame threshold
def FillBlackFrames(clip, thresh=0.1, debug=False):
if not isinstance(clip, vs.VideoNode):
raise ValueError('This is not a clip')

def selectFunc(n, f):
nonlocal clip
firstGoodFrame = f.props['FirstGoodFrame']
if n == 0:
clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=firstGoodFrame)
return clip
if firstGoodFrame >= n: # this frame, we already dealt with
return clip

if f.props['PlaneStatsAverage'] > thresh or n == 0:
if debug:
return core.text.Text(clip=clip, text="Org, avg: " + str(f.props['PlaneStatsAverage']), alignment=8)
clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=n)
return clip
firstFrametoReplace = n
firstGoodFrame = n + 1
for i in range(n, clip.num_frames-1):
if clip.get_frame(i).props['PlaneStatsAverage'] <= thresh:
continue
firstGoodFrame = i
break
clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=firstGoodFrame)
replaced = fillMultipleWithSVP(clip, firstFrametoReplace, firstGoodFrame, gpu=True)
if debug:
return core.text.Text(clip=replaced, text="Replaced from " + str(firstFrametoReplace) + " to " + str(firstGoodFrame-1))
return replaced
clip = core.std.SetFrameProp(clip, prop="FirstGoodFrame", intval=0)
clip = core.std.PlaneStats(clip)
fixed = core.std.FrameEval(clip, selectFunc, prop_src=clip)
return fixed



When I have a clip where:
Frame 19 is fine
Frame 20&21 are black
Frame 22 is fine
and I apply FillBlackFrames to it, both frame 20&21 get replaced, but frame 21 is the same as 22.

Does someone spot where I went wrong?
(Or is this caused by Vapoursynth not really supporting linear processing in script?)

Cu Selur

Latest version:on GitHub (https://github.com/Selur/VapoursynthScriptsInHybrid/blob/master/ReplaceBlackFrames.py)

[kedautinh12]

Why ReplaceBlackFrames same meaning with FillBlackFrames??

[_Al_]

I just tried that first one, ReplaceBlackFrames, that you say it works, but anyway to simplify it because all it has to be done is to remember last non black n for a frame. To "remember" things is always a flag to introduce a class that just handles "globals" within itself.
ReplaceBlackFrames:
import vapoursynth as vs
from vapoursynth import core

class RBF:
def __init__(self, clip, thresh=0.1, debug=False):
self.last_non_black_n = 0
self.clip = clip
self.thresh = thresh
self.debug = debug

def selectFunc(self, n, f):
if f.props['PlaneStatsAverage'] > self.thresh or n == 0:
out = self.clip
self.last_non_black_n = n
else:
out = self.clip[self.last_non_black_n]
if self.debug:
return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def ReplaceBlackFrames(clip, thresh=0.1, debug=False):
clip = core.std.PlaneStats(clip)
rbf = RBF(clip, thresh, debug)
fixed = core.std.FrameEval(clip, rbf.selectFunc, prop_src=clip)
return fixed

clip1 = core.std.BlankClip(color=(255,0,0)) #red
clip2 = core.std.BlankClip() #black
clip3 = core.std.BlankClip(color=(255,255,255)) #white
clip = clip1+clip2+clip3
clip = ReplaceBlackFrames(clip)
clip.set_output()
so to keep the same logic, FillBlackFramescould be constructed, you do not need to mess up with indices , trimming etc.

[_Al_]

to simplify it even more, interestingly, this works too:
import vapoursynth as vs
from vapoursynth import core

class ReplaceBlackFrames:
def __init__(self, clip, thresh=0.1, debug=False):
self.last_non_black_n = 0
self.clip = clip.std.PlaneStats(clip)
self.thresh = thresh
self.debug = debug

def selectFunc(self, n, f):
if f.props['PlaneStatsAverage'] > self.thresh or n == 0:
out = self.clip
self.last_non_black_n = n
else:
out = self.clip[self.last_non_black_n]
if self.debug:
return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

@property
def out(self):
return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip)

clip1 = core.std.BlankClip(color=(255,0,0)) #red
clip2 = core.std.BlankClip() #black
clip3 = core.std.BlankClip(color=(255,255,255)) #white
clip = clip1+clip2+clip3
rbf = ReplaceBlackFrames(clip, debug=True)
rbf.out.set_output()

[_Al_]

OK, :-), just tested it, it works in a linear frame request fashion, like enkoding etc., but in a previewer with random frame access it needs to be like Selur posted, hunting for a previous good frame in a loop (maybe there is another way, not sure):
import vapoursynth as vs
from vapoursynth import core

class ReplaceBlackFrames:
def __init__(self, clip, thresh=0.1, debug=False):
self.clip = clip.std.PlaneStats(clip)
self.thresh = thresh
self.debug = debug

def selectFunc(self, n, f):
out = self.clip[self.get_good_n(n)]
if self.debug:
return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def get_good_n(self, n):
for i in reversed(range(n+1)):
if self.clip.get_frame(i).props['PlaneStatsAverage'] > self.thresh:
return i
else:
return n

@property
def out(self):
return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip)

clip1 = core.std.BlankClip(color=(255,0,0)) #red
clip2 = core.std.BlankClip() #black
clip3 = core.std.BlankClip(color=(255,255,255)) #white
clip = clip1+clip2+clip3
rbf = ReplaceBlackFrames(clip, debug=True)
rbf.out.set_output()

[Selur]

@_AI_:Thanks, will look at it later after work.
@kedautinh12: Why ReplaceBlackFrames same meaning with FillBlackFrames??
Yeah, naming could be better, it might end up with just ReplaceBlackFrames and a 'method'-parameter ('last non-black','interpolation (SVP)',...).

[_Al_]

As for FillBlackFrames, to interpolate black frames,
I came up with this, seams to work, but tested only with posted example, where I manually inserted 5 black frames into a video (made sure there was some movement) and it was successfully interpolated:

import vapoursynth as vs
from vapoursynth import core

class FillBlackFrames:
def __init__(self, clip, thresh=0.1, debug=False, gpu=False):
self.clip = clip.std.PlaneStats(clip)
self.thresh = thresh
self.debug = debug
self.gpu = gpu
self.smooth = None

def selectFunc(self, n, f):
out = self.get_clip(n)
if self.debug:
return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def is_not_black(self, n):
return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh

def get_clip(self, n):
if self.is_not_black(n):
return self.clip[n]

for start in reversed(range(n+1)):
if self.is_not_black(start):
break
else:
#there are all black frames preceding n, return current n frame
return self.clip[n]

for end in range(n, len(self.clip)):
if self.is_not_black(end):
break
else:
#there are all black frames to the end, return current n frame
return self.clip[n]

#does interpolated smooth clip exists for requested n frame? Use n frame from it.
if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end:
return self.smooth[n-start]

#interpolating two frame clip into end-start+1 fps
clip = self.clip[start] + self.clip[end]
clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1)
if self.gpu:
super = core.svp1.Super(clip,"{gpu:1}")
else:
super = core.svp1.Super(clip,"{gpu:0}")
vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}")
num = end - start +1
self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}")
self.smooth_start = start
self.smooth_end = end
return self.smooth[n-start]

@property
def out(self):
return core.std.FrameEval(self.clip, self.selectFunc, prop_src=self.clip)

clip1 = core.lsmas.LibavSMASHSource(r'F:\video.mp4')[500:530]
clip2 = clip1.std.BlankClip()[:5] # inserted 5 black frames
clip3 = core.lsmas.LibavSMASHSource(r'F:\video.mp4')[535:700]

clip = clip1+clip2+clip3
fbf = FillBlackFrames(clip, debug=True)
fbf.out.set_output()
It interpolates from one available frame on the left and one to the right.

Does it interpolate better if it has more frames available on those sides? I guess yes, I might add it later , to use more frames if it is possible.

[Selur]

Hmm,.. I changed it to:
import vapoursynth as vs
from vapoursynth import core

'''
call using:

from ReplaceBlackFrames import ReplaceBlackFrames
rbf = ReplaceBlackFrames(clip, debug=True, thresh=0.1, method='previous')
clip = rbf.out

debug: whether to display the average luma
method:
'previous': replace black frames with the last non-black frame
'interpolateSVP': replace black frames whit interpolatied frames using SVP (GPU)
'interpolateSVPCPU': replace black frames whit interpolatied frames using SVP (CPU)

'''

class ReplaceBlackFrames:
# constructor
def __init__(self, clip: vs.VideoNode, thresh: float=0.1, debug: bool=False, method: str='previous'):
self.clip = clip.std.PlaneStats(clip)
self.thresh = thresh
self.debug = debug
self.method = method

def previous(self, n, f):
if f.props['PlaneStatsAverage'] > self.thresh or n == 0:
out = self.clip
self.last_non_black_n = n
else:
out = self.clip[self.last_non_black_n]
if self.debug:
return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def interpolate(self, n, f):
out = self.get_clip(n)
if self.debug:
return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def is_not_black(self, n):
return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh

def get_clip(self, n):
if self.is_not_black(n):
return self.clip[n]

for start in reversed(range(n+1)):
if self.is_not_black(start):
break
else: #there are all black frames preceding n, return current n frame
return self.clip[n]

for end in range(n, len(self.clip)):
if self.is_not_black(end):
break
else:
#there are all black frames to the end, return current n frame
return self.clip[n]

#does interpolated smooth clip exists for requested n frame? Use n frame from it.
if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end:
return self.smooth[n-start]

#interpolating two frame clip into end-start+1 fps
clip = self.clip[start] + self.clip[end]
clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1)
if self.method == 'interpolateSVP':
super = core.svp1.Super(clip,"{gpu:1}")
else:
super = core.svp1.Super(clip,"{gpu:0}")
vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}")
num = end - start +1
self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}")
self.smooth_start = start
self.smooth_end = end
return self.smooth[n-start]

@property
def out(self):
if self.method == 'previous':
return core.std.FrameEval(self.clip, self.previous, prop_src=self.clip)
else:
return core.std.FrameEval(self.clip, self.interpolate, prop_src=self.clip)

to join both methods (replace and interpolate) into one class (don't think I changed the logic).

Sadly, only the 'previous' (= old ReplaceBlackFrames) work. Using 'interpolateSVP' I end with the output not having interpolated frames.
No clue where I broke it. :(

I uploaded my test file and my test script to my GoogleDrive (https://drive.google.com/drive/folders/1sJfL-p2J-Ouvx_y42R2Oa7es9w_FGx95?usp=share_link).

Cu Selur

[_Al_]

In constructor method, __init__(), add
self.smooth=None
and two else blocks are in a wrong block, notice in my example:

for start in reversed(range(n+1)):
if self.is_not_black(start):
break
else:
#there are all black frames preceding n, return current n frame
return self.clip[n]

for end in range(n, len(self.clip)):
if self.is_not_black(end):
break
else:
#there are all black frames to the end, return current n frame
return self.clip[n]
if for in loop is exhausted, going thru whole range and it is not breaking, would fire else statement. So that else belongs under for in loop, not below that if block statement.

Also for that simple previous good frame instead of black, you selected version that works the same only in linear fashion, I guess that second should be used.

[_Al_]

I rather post your integrated solution, the whole thing, as it should be I guess, so there is no confusion:

import vapoursynth as vs
from vapoursynth import core

'''
call using:

from ReplaceBlackFrames import ReplaceBlackFrames
rbf = ReplaceBlackFrames(clip, debug=True, thresh=0.1, method='previous')
clip = rbf.out

debug: whether to display the average luma
method:
'previous': replace black frames with the last non-black frame
'interpolateSVP': replace black frames whit interpolatied frames using SVP (GPU)
'interpolateSVPCPU': replace black frames whit interpolatied frames using SVP (CPU)

'''

class ReplaceBlackFrames:
# constructor
def __init__(self, clip: vs.VideoNode, thresh: float=0.1, debug: bool=False, method: str='previous'):
self.clip = core.std.PlaneStats(clip)
self.thresh = thresh
self.debug = debug
self.method = method
self.smooth = None

def previous(self, n, f):
out = self.get_current_or_previous(n)
if self.debug:
return out.text.Text(text="Org, avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def interpolate(self, n, f):
out = self.get_current_or_interpolate(n)
if self.debug:
return out.text.Text(text="avg: "+str(f.props['PlaneStatsAverage']),alignment=8)
return out

def get_current_or_previous(self, n):
for i in reversed(range(n+1)):
if self.is_not_black(i):
return self.clip[i]
else:
#all previous are black, return current n frame
return self.clip[n]

def get_current_or_interpolate(self, n):
if self.is_not_black(n):
#current non black selected
return self.clip[n]

#black frame, frame is interpolated
for start in reversed(range(n+1)):
if self.is_not_black(start):
break
else: #there are all black frames preceding n, return current n frame
return self.clip[n]

for end in range(n, len(self.clip)):
if self.is_not_black(end):
break
else:
#there are all black frames to the end, return current n frame
return self.clip[n]

#does interpolated smooth clip exist for requested n frame? Use n frame from it.
if self.smooth is not None and start >= self.smooth_start and end <= self.smooth_end:
return self.smooth[n-start]

#interpolating two frame clip into end-start+1 fps
clip = self.clip[start] + self.clip[end]
clip = clip.std.AssumeFPS(fpsnum=1, fpsden=1)
if self.method == 'interpolateSVP':
super = core.svp1.Super(clip,"{gpu:1}")
else:
super = core.svp1.Super(clip,"{gpu:0}")
vectors = core.svp1.Analyse(super["clip"],super["data"],clip,"{}")
num = end - start + 1
self.smooth = core.svp2.SmoothFps(clip,super["clip"],super["data"],vectors["clip"],vectors["data"],f"{{rate:{{num:{num},den:1,abs:true}}}}")
self.smooth_start = start
self.smooth_end = end
return self.smooth[n-start]

def is_not_black(self, n):
return self.clip.get_frame(n).props['PlaneStatsAverage'] > self.thresh

@property
def out(self):
if self.method == 'previous':
return core.std.FrameEval(self.clip, self.previous, prop_src=self.clip)
else:
return core.std.FrameEval(self.clip, self.interpolate, prop_src=self.clip)

[Selur]

Nice! Thanks a lot. This confused about how Python uses indention. :)
... Ahhh I got it, the 'else: ' part are executed if the for loop ends normally (not stopped by break)

Cu Selur

Ps.: but it on GitHub (https://github.com/Selur/VapoursynthScriptsInHybrid/blob/master/ReplaceBlackFrames.py) and added interpolation through RIFE to it.

[_Al_]

more smooth results are using:
num = end - start
if using num = end - start + 1 there seams to be a slight jump in position. Not sure about RIFE, just using SVP.

[Selur]

I'll test it. :)

[Selur]

Works with RIFE fine too. :)

Cu Selur