Duplicity2/DropDeadGorgeous v2.13 - Dupe Tool - 17 Nov 2018

[StainlessS]

Should work fine as avsi in plugs directory.

[StainlessS]

Duplicity2, v2.10, new version, see 1st post.

Main diffs.

   Function Duplicity2(clip c,"Mode"=0,Float "ThG"=1.0,Float "ThB"=ThG*2.0, Int "ThP"=64,Float "IgP"=0.0,Int "ThK"=7,
        \ Int "MaxDupLen"=9,Int "LagMax"=MaxDupLen,Int "MaxInterp"=Min(9,MaxDupLen),
        \ int "BlkW"=64,int "BlkH"=BlkW,int "OLapX"=BlkW/2,Int "OLapY=BlkH/2",
        \ Int "X"=0,Int "Y"=0,Int"W"=0,Int "H"=0,Float "CW"=0.0,Bool "ChromaI"=False,Bool "AltScan"=False,
        \ Int "SPad"=16, Int "SSharp"=1, Int "SRFilter"=4,                             [* MSuper          *]
        \ Int "ABlkSize"=16, Int "AOverlap"=4,   Int "ASearch"=3,Int "ADct"=0,         [* MAnalyse        *]
        \ Int "RBlkSize"=8,  Int "ROverlap"=2,   Int "RthSAD"=100,                     [* MRecalculate    *]
        \ Float "Iml"=200.0, Bool "IBlend"=True, Int "IthSCD1"=400, Int "IthSCD2"=130, [* MFlowInter      *]
        \ Int "OthSCD1"=400,Int "OthSCD2"=130,                                         [* Dupe Detect Override @ Scene Change *]
        \ String "OverRide"="",String "Frames"="Frames.txt",Bool "Ranges"=True,Bool "FrameAsRange"=False,
        \ Bool "Show"=True,Bool "Verb"=True,Bool "ShowBlk"=True,Bool "Dim"=True,Bool "ShowDot"=False,
        \ Bool "InterpFast"=True,String "DBase"=""
        \ )

        InterpFast,   Default True. If True, then Interpolation clips are pre-calculated, if false then calculations done in real time.
                      Pre-Calc clips a bit (maybe 10%) faster, but perhaps not quite as good quality.

        DBase,        Default "", If named DBase suppled, then is not deleted on Function exit.
                          [ If name supplied, result DB may be re-used by other scripts, eg DropDeadGorgeous() function ].
                      NOTE, If using multiple instances and named DBase, then give each DBase as different name.

[StainlessS]

Also see previous post (needed by this script).

Here its play time,

/*

    DropDeadGorgeous() v -1.0, Beta -1,  by the Gorgeous & Truly Scrumptious StainlessS @ Doom9

        Using DBase as Created by Duplicity2 v2.10,
        Corrects problems where frames were dropped in clip, and then some time later duplicates were inserted to maintain timing.
        ScanAhead is number of frames scanned ahead of current frame looking for duplicates, currently no maximum, default 9.
        If duplicates found (no current limit on length of duplicate run), then biggest frame jump (movement) prior to dupes is
        located, and a number of frames interpolated prior to it (same number as in dupe run).
        Fixes jumpy jaunty clips that constantly and repeatedly jump and then freeze.

        Simple metrics on Show, "nnn] CIR", where nnn is frame number, and character Flags highlited where,
            C=Copy frame (original as source frame)
            I=Interpolated frame
            R=Relocated frame from earlier in clip (interpolated frames shove following frames to later in clip, removing dupes)

        Can use Duplicity Modes 1 or 2 in creating DBase.

*/

Function DropDeadGorgeous(clip c,String DB,Int "ScanAhead",
    \ Int "X",Int "Y",Int"W",Int "H",
    \ Int "SPad", Int "SSharp", Int "SRFilter",                         [* MSuper          *]
    \ Int "ABlkSize", Int "AOverlap", Int "ASearch",Int "ADct",         [* MAnalyse        *]
    \ Int "RBlkSize", Int "ROverlap", Int "RthSAD",                     [* MRecalculate    *]
    \ Float "Iml", Bool "IBlend", Int "IthSCD1", Int "IthSCD2",         [* MFlowInter      *]
    \ Bool "Show"
    \ ){
    c   myName="DropDeadGorgeous: "
    IsAvsPlus=(FindStr(UCase(versionString),"AVISYNTH+")!=0)    HasGScript=RT_FunctionExist("GScript")
    HasGRunt =RT_FunctionExist("GScriptClip")                   HasMvTools=RT_FunctionExist("MSuper")
    HasMaskTools=RT_FunctionExist("MT_Lutxy")                   HasRemoveGrain=RT_FunctionExist("RemoveGrain")
    HasCallCmd=RT_FunctionExist("CallCmd")
    ScanAhead = Default(ScanAhead,9)
    X=default(X,0)                      Y=default(Y,0)                      W=default(W,0)                    H=default(H,0)
    SPad      = Default(SPad,16)        SSharp  = Default(SSharp,1)         SRFilter= Default(SRFilter,4)
    ABlkSize  = Default(ABlkSize,16)    AOverlap= Default(AOverlap,4)       ASearch = Default(ASearch,3)      ADct   = Default(ADct,0)
    RBlkSize  = Default(RBlkSize,8)     ROverlap= Default(ROverlap,2)       RthSAD  = Default(RthSAD,100)
    Iml       = Default(Iml,200.0)      IBlend  = Default(IBlend,True)      IthSCD1 = Default(IthSCD1,400)    IthSCD2= Default(IthSCD2,130)
    Show      = Default(Show,False)
    Assert(0 < ScanAhead,RT_String("%s0 < ScanAhead(%d)",myName,ScanAhead))
    ####
    Assert(DB!="",RT_String("%sDB Cannot be ''",myName))
    DB1 = RT_GetFullPathName(DB)
    DB2 = RT_GetFullPathName("~@@@_"+RT_LocalTimeString+".DB")    RT_DBaseAlloc(DB2,FrameCount,"iii")
    W = (W > 0) ? (W + X) - Width  : W     H = (H > 0) ? (H + Y) - Height : H    # Make -ve relative width,height
    X = (X+3) / 4 * 4                      Y = (Y+3)  / 4 * 4                    # Round UP  (inwards) Mod 4
    W = (W-3) / 4 * 4                      H = (H-3)  / 4 * 4                    # Round DOWN(inwards) Mod 4
    W = Width - X + W                      H = Height - Y + H                    # +ve width,height
    DC=Crop(X,Y,W,H)
    Global G_DIFMOD=1   # Tmp setting
    FuncS="""
        Function Fn@@@(clip c,clip dc,String DB1,String DB2,int ScanAhead,Bool Show) {
            c    n=current_frame
            Stat2 = RT_DBaseGetField(DB2,n,0)
            if(Stat2 == 0) {
                Stat1 = RT_DBaseGetField(DB1,n,FLD_STAT@@@)
                if(Stat1 < FLG_DE@@@ && SOS@@@.RT_AverageLuma(n=n,w=1,h=1).Int == 0) { # Non dupe and NOT Start Of Scene
                    de = Min(n+(ScanAhead*2),FrameCount-1)
                    DupN = -1
                    for(i=n+1,de) { if(RT_DBaseGetField(DB1,i,FLD_STAT@@@) >= FLG_DE@@@) { DupN = i i=de } }
                    if(DupN<0) {
                        ds=Max(de-ScanAhead,n+1)
                        for(i=n,ds-1) { RT_DBaseSetField(DB2,i,0,1) }   # Copy
                    } Else { # Both n and DupN, NOT SOS
                        ds = Max(DupN - ScanAhead, n)
                        for(i=DupN-1,ds, -1) {
                            if(SOS@@@.RT_AverageLuma(n=i,w=1,h=1).Int != 0) { tmp=i  i=ds  ds=tmp+1}
                        }
                        for(i=n,ds-1) { RT_DBaseSetField(DB2,i,0,1) }   # Copy
                        if(ds<DupN) {
                            Index=RT_DBaseGetField(DB1,DupN,FLD_INDEX@@@)
                            SSTart=DupN-Index
                            Count=RT_DBaseGetField(DB1,SSTart,FLD_COUNT@@@)     # Dup Count stored only in first Record of Sub Sequence
                            mxi    = ds
                            if(G_DIFMOD==0)             { mxdif = MFPC@@@.RT_AverageLuma(n=ds)          }
                            Else                        { mxdif = dc.RT_LumaDifference(dc,n=ds,n2=ds-1) }
                            for(i=ds+1,DupN-1) {
                                if(G_DIFMOD==0)         { dif = MFPC@@@.RT_AverageLuma(n=i)             }
                                Else                    { dif = dc.RT_LumaDifference(dc,n=i,n2=i-1)     }
                                if(dif>mxDif) {
                                    mxDif=dif
                                    mxi = i
                                }
                            }
                            for(i=mxi+Count,DupN+Count-1) {
                                RT_DBaseSetField(DB2,i,0,2)         # Copy Frame relative
                                RT_DBaseSetField(DB2,i,1,COUNT)     # Copy Frame
                            }
                            for(i=mxi,mxi+Count-1) {
                                RT_DBaseSetField(DB2,i,0,3)         # Interp
                                RT_DBaseSetField(DB2,i,1,COUNT)     # Interp count
                                RT_DBaseSetField(DB2,i,2,i-mxi)     # Index
                            }
                            for(i=ds,mxi-1) {
                                RT_DBaseSetField(DB2,i,0,1)
                            }  # copy
                        }
                    }
                } Else { RT_DBaseSetField(DB2,n,0,1) } # Dupe or Start Of Scene
                Stat2 = RT_DBaseGetField(DB2,n,0)
            }
            if(Stat2==1) {              # Copy (already correct frame)
                Stat1 = RT_DBaseGetField(DB1,n,0)
                if(Show) { RT_Subtitle("%d] C\aLIR",n) }
            } Else if(Stat2==2) {       # Copy previous relative
                Count = RT_DBaseGetField(DB2,n,1)
                Stat1 = RT_DBaseGetField(DB1,n,0)
                c.Trim(n-Count,-1)
                if(Show) { RT_Subtitle("%d] \aLCI\a-R",n) }
            } Else if(Stat2==3) {
                Index = RT_DBaseGetField(DB2,n,2)
                Count = RT_DBaseGetField(DB2,n,1)
                SStart= n-Index
                fillMax=3
                LS=Max(SStart-fillMax,0)  RE=Min(SStart+FillMax-1,c.FrameCount-1)
                LCnt=SStart-LS
                Tmp_c = c.Trim(LS,-(RE-LS+1))
                Prefilt=Tmp_c.RemoveGrain(22)
                Super=Tmp_c.MSuper(hpad=SPad@@@,vpad=SPad@@@,levels=1,sharp=SSharp@@@,rfilter=SRFilter@@@)
                Superfilt=Prefilt.MSuper(hpad=SPad@@@,vpad=SPad@@@,sharp=SSharp@@@,rfilter=SRFilter@@@)
                bv=Superfilt.MAnalyse(isb=true  ,blksize=ABlkSize@@@,overlap=AOverLap@@@,search=ASearch@@@,delta=1,dct=ADct@@@)
                fv=Superfilt.MAnalyse(isb=False ,blksize=ABlkSize@@@,overlap=AOverLap@@@,search=ASearch@@@,delta=1,dct=ADct@@@)
                if(RBlkSize@@@>0) {
                    bv=Super.MRecalculate(bv,blksize=RBlkSize@@@,overlap=ROverLap@@@,thSAD=RthSAD@@@)
                    fv=Super.MRecalculate(fv,blksize=RBlkSize@@@,overlap=ROverLap@@@,thSAD=RthSAD@@@)
                }
                Tmp_I=Tmp_c.MFlowInter(Super,bv,fv,time=100.0*(Index+1)/(Count+1),ml=Iml@@@,blend=IBlend@@@,thSCD1=IthSCD1@@@,thSCD2=IthSCD2@@@)
                Tmp_I.Trim(LCnt-1,-1)  # NO SHIFT, fixed frame @ LCnt-1 ie same as Left PreDupe frame
                Stat1 = RT_DBaseGetField(DB1,n,0)
                if(Show) { RT_Subtitle("%d] \aLC\a-I\aLR",n) }
            } Else {
                RT_Subtitle("%d] %d %d BUG BUG BUG",n,Stat1,Stat2)
            }
            Return Last
        }
        Function StartOfSceneClip(clip c,Int "thSCD1",Int "thSCD2") { # All Luma Samples set 255 at SOS, else 0
            thSCD1=Default(thSCD1,400)  thSCD2=Default(thSCD2,130)
            sup=c.MSuper(pel=1,sharp=0,rfilter=2,hpad=16, vpad=16)
            fv=sup.MAnalyse(isb=false,delta=1,blksize=16)
            Return c.MSCDetection(fv,thSCD1=thSCD1,thSCD2=thSCD2)
        }
        Function MMaskFromPrevClip(clip c,Int "MaskType",Float "Gamma",Int "thSCD1",Int "thSCD2") {
            # MaskType:- 0=Motion, 1=SAD, 2=Occlusion, 3=Horizontal, 4=Vertical, 5=ColorMap.
            MaskType=Default(MaskType,0)  Gamma=Default(Gamma,1.0)  thSCD1=Default(thSCD1,400)  thSCD2=Default(thSCD2,130)
            sup=c.MSuper(pel=1,sharp=0,rfilter=2,hpad=16, vpad=16)
            fv=sup.MAnalyse(isb=false,delta=1,blksize=16)
            Return c.MMask(fv,Gamma=1.0,kind=MaskType,thSCD1=thSCD1,thSCD2=thSCD2)
        }
        Global SOS@@@ = dc.StartOfSceneClip(thSCD1=IthSCD1,thSCD2=IthSCD2)
        Global MFPC@@@= dc.MMaskFromPrevClip(Gamma=2.0,MaskType=0,thSCD1=IthSCD1,thSCD2=IthSCD2)
        ### DB1 Fields:-
        Global FLD_STAT@@@   = 0   # Holds Status FLGS
        Global FLD_START@@@  = 1   # Unique Start Frame
        Global FLD_END@@@    = 2   # Unique End   Frame
        Global FLD_ANCHOR@@@ = 3   # Anchor Frame
        Global FLD_INDEX@@@  = 4   # Index into Count (0 Relative, 0->Count-1)
        Global FLD_COUNT@@@  = 5   # Sequence Frame Count
        ### DB1 Status Flags in Field FLD_STAT@@@
        Global FLG_OO@@@     = 1   # User Override
        Global FLG_UK@@@     = 2   # K Override
        Global FLG_UU@@@     = 3   # Unique Frame
        Global FLG_NUK@@@    = 4   # Non Unique Sequence K Override
        Global FLG_LM@@@     = 5   # LoMotion Frame
        Global FLG_DS@@@     = 6   # Static Sequence [detected as duplicates but Longer than MaxDupLen]
        Global FLG_DE@@@     = 7   # Duplicate Sequence is followed by Scene Change (will NOT occar when Mode==1)
        Global FLG_DUP@@@    = 8   # Duplicate Sequence
        ###
        Global SPad@@@=SPad           Global SSharp@@@=SSharp       Global SRFilter@@@=SRFilter
        Global ABlkSize@@@=ABlkSize   Global AOverLap@@@=AOverLap   Global ASearch@@@=ASearch        Global ADct@@@=ADct
        Global RBlkSize@@@=RBlkSize   Global ROverLap@@@=ROverLap   Global RthSAD@@@=RthSAD
        Global Iml@@@=Iml             Global IBlend@@@=IBlend       Global IthSCD1@@@=IthSCD1        Global IthSCD2@@@=IthSCD2
        ###
        ARGS = "dc,DB1,DB2,ScanAhead,Show"
        ScriptLine = "Fn@@@("+ARGS+")"
        Last.GScriptClip(ScriptLine, local=true, args=ARGS)
    """
    GIFunc="DROPDEADGORGEOUS"                   # Function Name, Supply unique name for your multi-instance function.
    GIName=GIFunc+"_InstanceNumber"    # Name of the Instance number Global
    RT_IncrGlobal(GIName)              # Increment Instance Global (init to 1 if not already exists)
    GID   = GIFunc + "_" + String(Eval(GIName))
    InstS = RT_StrReplace(FuncS,"@@@","_"+GID)
    # Write Debug GScript File to current directory (line numbers will correspond to error messages)
#   RT_WriteFile("DEBUG_"+GID+".TXT","%s",InstS)
    HasGScript ? GScript(InstS) : Eval(InstS,"GS_EVAL") # Use GSCript if installed (loaded plugs override builtin)
    # if CallCmd available, Auto delete DBase file on clip closure.
    HasCallCmd?CallCmd(close=RT_String("""CMD /C chcp 1252 && del "%s" """,DB2), hide=true, Synchronous=7):NOP
    Return Last
}

FN  = "D:\G.AVI"    DBase = "D:\G_Det.DB"     # DBase Created by Duplicity2
#FN = "D:\HAR.AVI"  DBase = "D:\HAR_Det.DB"
Avisource(FN)
ORG=Last
SHOW=True
DropDeadGorgeous(DBase,show=Show)
StackVertical(Last,ORG)

EDIT: Fixed I flag, was not hilted.

Strictly sequential, play forwards ONLY.

[StainlessS]

Duplicity2 v2.11 bugfix update, see first post.

Two bugs, first one probably a copy paste of existing bug, suspect other scripts might suffer from same bug.

Was

    X = (X+3) / 4 * 4                      Y = (Y+3) / 4 * 4                       # Round UP Mod 4
    W = (W<=0) ? Width() - X + W  : W      H = (H<=0) ? Height() - Y + H : H
    W = W / 4 * 4                          H = H / 4 * 4                           # Round DOWN Mod 4

Fixed

    W = (W > 0) ? (W + X) - Width  : W     H = (H > 0) ? (H + Y) - Height : H    # Make -ve relative width,height
    X = (X+3) / 4 * 4                      Y = (Y+3)  / 4 * 4                    # Round UP  (inwards) Mod 4
    W = (W-3) / 4 * 4                      H = (H-3)  / 4 * 4                    # Round DOWN(inwards) Mod 4
    W = Width - X + W                      H = Height - Y + H                    # +ve width,height

Rounding up X mod 4 should also have affected width, but did not, I always use -ve width, height (as for crop) where problem does not arise.
Same fix applied to previous post above, DropDeadGorgeous() script.

2nd bug, in Client script

Import(".\Duplicity2.Avsi")


######
FN  = "D:\G.AVI"    DBase  = "D:\G_Det.DB"       ThB=2.0
#FN = "D:\HAR.AVI"   DBase  = "D:\HAR_Det.DB"    ThB=2.5
######
Avisource(FN)
SRC=Last
###
Mode=2             # 0=Write Frames only : 1=Replace Dupes with Exact Dupes : 2= Interpolate/Blend Duplicates.
ThG=1.0            # Primary Frame Global Detect
ThP=64             # Pixel Count Detect
MaxDupLen=9        # Max detected, if more then is considered static sequence not duplicate)
MaxInterp=9        # Max Interpolated frames when Mode=2 (More than MaxInterp & less or equal to MaxDupLen then uses Blend mode)
InterpFast=true
Show=True
ShowDot=False
ShowBlk=True
VERB=True
###
TITLE=true
STACK=False
STACK4=True
###

Result=SRC.Duplicity2(Mode=Mode,ThG=ThG,ThB=ThB,ThP=ThP,MaxDupLen=MaxDupLen,MaxInterp=MaxInterp,
    \ Show=False,InterpFast=InterpFast,DBase=DBase)

Metrics=SRC.Duplicity2(Mode=Mode,ThG=ThG,ThB=ThB,ThP=ThP,MaxDupLen=MaxDupLen,MaxInterp=MaxInterp,
    \ Show=Show,Verb=VERB,ShowBlk=ShowBlk,ShowDot=ShowDot,InterpFast=InterpFast)

Dif=Subtract(Result,SRC)

SRC     = (TITLE) ? SRC.TSub("Src")            : SRC
Result  = (TITLE) ? Result.TSub("Result")       : Result
Metrics = (TITLE) ? Metrics.TSub("Metrics")     : Metrics
Dif     = (TITLE) ? Dif.TSub("Dif(Result,Src)") : Dif

LFT=StackVertical((STACK4||!SHOW)?Result:Metrics,SRC)
RGT=StackVertical(Metrics,Dif)
STK4=StackHorizontal(LFT,RGT)

(STACK4) ? STK4 : (STACK) ? LFT : (Show)?Metrics:Result

Return Last


# Stack Overhead Subtitle Text, with optional FrameNumber shown.
Function TSub(clip c,string Tit,Bool "ShowFrameNo",Int "Col"){
    c.BlankClip(height=20,Color=Default(Col,0))
    (Default(ShowFrameNo,False))?ScriptClip("""Subtitle(String(current_frame,"%.f] """+Tit+""""))"""):Subtitle(Tit)
    Return StackVertical(c).AudioDubEx(c)
}

Moved DBase to result clip call instead of metrics clip (would not create named DBase if not showing metrics).

NOTE. I have found a problem in AVS+ (dont happen in Avs 2.6 standard), not sure if avs+ bug or not.
With all metrics showing and also SHOWBLK=True, and INTERPFAST=True, after playing about 1000 or so frames,
go backwards a number of frames and you get out of memory error. Does not happen on AVS standard, or with InterpFast=False.
Possibly a different system resources requirement of Avs+ (dont know). [Tested using VDub2 and VDubMod, Win7 x64 Avs/+ 32bit]

I've also noticed that ClipClop() included DoctorFrames script now fails at MaxInterp of 15 for Avs+ where using same method as used in Duplicity2
when InterpFast=True. Avs 2.6 standard achieves about 20 sets of precalculated clips before failing (with SetMemoryMax could get as hi as 22).

EDIT: Zip on mediafire/SendSpace updated, scripts from posts #22 updated
(also version number update @ top of 1st doc page on post #1).

EDIT: This is the pre-calculated set of interpolation clips where INTERPFAST=True in duplicity2, and similar used in ClipClop DoctorFrames().

        if(Mode==2 && MaxInterp>0) {
            if(InterpFast) {
                Prefilt  = c.RemoveGrain(22)
                Super    = c.MSuper(hpad=SPad,vpad=SPad,levels=1,sharp=SSharp,rfilter=SRFilter) # One level is enough for MRecalculate
                Superfilt= Prefilt.MSuper(hpad=SPad,vpad=SPad,sharp=SSharp,rfilter=SRFilter)    # All levels for MAnalyse
                For(i=1, MaxInterp) {
                    Eval(RT_String("bv=Superfilt.MAnalyse(isb=true ,blksize=ABlkSize,overlap=AOverLap,search=ASearch,delta=%d,dct=ADct)",i+1))
                    Eval(RT_String("fv=Superfilt.MAnalyse(isb=false,blksize=ABlkSize,overlap=AOverLap,search=ASearch,delta=%d,dct=ADct)",i+1))
                    if(RBlkSize>0) {
                        Eval(RT_String("bv=Super.MRecalculate(bv,blksize=RBlkSize,overlap=ROverLap,thSAD=RthSAD)"))
                        Eval(RT_String("fv=Super.MRecalculate(fv,blksize=RBlkSize,overlap=ROverLap,thSAD=RthSAD)"))
                    }
                    for(j=1,i) {
                        Eval(RT_String("Global I%d%d@@@=c.MFlowInter(Super,bv,fv,time=100.0*%d/%d," +
                            \ "ml=Iml,blend=IBlend,thSCD1=IthSCD1,thSCD2=IthSCD2).SelectEvery(1,%d).Trim(0,-c.FrameCount)", i,j-1, j,i+1,-j))
                    }
                }
            } Else {
                Global SPad@@@=SPad           Global SSharp@@@=SSharp       Global SRFilter@@@=SRFilter
                Global ABlkSize@@@=ABlkSize   Global AOverLap@@@=AOverLap   Global ASearch@@@=ASearch        Global ADct@@@=ADct
                Global RBlkSize@@@=RBlkSize   Global ROverLap@@@=ROverLap   Global RthSAD@@@=RthSAD
                Global Iml@@@=Iml             Global IBlend@@@=IBlend       Global IthSCD1@@@=IthSCD1        Global IthSCD2@@@=IthSCD2
            }
        }

EDIT: Could not get Avs+ to fail where STACK and STACK4 both false in client script, ie where only single instance of Duplicity2 called [EDIT: Displayed].

EDIT: Updated Duplicty2 Client script a little bit in post 25 (No zip update).

[StainlessS]

Duplicity2 v2.12, update, see first post.

Zip now included DropDeadGorgeous() script versioned at v0.00.

SCRIPT REMOVED.

[StainlessS]

Duplicity2(), DropDeadGorgeous(), v2.13 updates, see first post.

See post #22 for current Duplicity2 script, or download see 1st post (re-directs to MediaFire or SendSpace below this post).

Current DropDeadGorgeous() reversioned to v2.13, will now always be same version as Duplicity2 (and require similar version Duplicity2).

DropDeadGorgeous.avsi part 1

/*

    DropDeadGorgeous() v2.13, by the Gorgeous & Truly Scrumptious StainlessS @ Doom9
    Same Requirements as Duplicity2(). https://forum.doom9.org/showthread.php?t=175357

        Using DBase as Created by Duplicity2 v2.13.
        Corrects problems where frames were dropped in clip, and then some time later duplicates were inserted to maintain timing,
        ie 'jumpy' clips that repeatedly jerk and then freeze.
        ScanAhead is number of frames scanned ahead of current_frame looking for duplicates, default 9. Any detected dupe run will
        start between current_frame and current_frame + ScanAhead.
        If duplicates found (limit on length of duplicate run set in Duplicity2 via MaxDupLen), then a JerkFrame is found.
        A JerkFrame is searched for backwards, from DupeStartFrame-1 to to current_frame and search stops if a StartOfScene is detected.
        The JerkFrame is the frame prior to first dupe, with greatest movement/jerk when compared with its predecessor.
        If JerkFrame is found, a number of frames are interpolated starting at the JerkFrame position, (same number as in dupe run,
        and source frames to interpolation are JerkFrame and JerkFrame-1), interpolated frames shove following frames to later in clip,
        removing dupes.
        If no JerkFrame is found, ie where duplicates are of a StartOfScene frame, then so long as DupeEndFrame+1 is not another StartOfScene
        frame, then duplicates are interpolated using DupeStartFrame-1 and DupeEndFrame+1 as interpolation source frames.

    Function DropDeadGorgeous(clip c,String DB,Int "ScanAhead"=9,
        \ Int "X"=0,Int "Y"=0,Int"W"=0,Int "H"=0,
        \ Int "SPad"=16, Int "SSharp"=1, Int "SRFilter"=4,                             [* MSuper          *]
        \ Int "ABlkSize"=16, Int "AOverlap"=4,   Int "ASearch"=3,Int "ADct"=0,         [* MAnalyse        *]
        \ Int "RBlkSize"=8,  Int "ROverlap"=2,   Int "RthSAD"=100,                     [* MRecalculate    *]
        \ Float "Iml"=200.0, Bool "IBlend"=True, Int "IthSCD1"=400, Int "IthSCD2"=130, [* MFlowInter      *]
        \ Bool "Show"=True, Bool "Verb"=True,Int "SOSthSCD2"=130
        \ )

        DB,            DBase as created by Duplicity2, v2.13, error if wrong version DBase.

        ScanAhead,     Default 9, 1 <= ScanAhead <= 60. Distance in frames to scan ahead of current frame. If first duplicate is always eg
                       3 frames after a jerk, then set this to 3.
                       ScanAhead, is deliberatley set quite high by default, then maybe reduce ScanAhead to MostCommonScanDist after full scan.
                       ALSO SEE Verbose Metrics MostCommonScanDist.

        X,Y,W,H,       All Default 0, Coordinates of test area, as in crop (all 0 = full frame).
                       The x,y coords will be rounded up mod4, and w,h coords rounded down mod 4.
                       Ignore eg animated logo.

                       MvTools2 Args, for Interpolation.
                       MSuper args:
        SPad,          Default 16,   See MvTools MSuper(hpad=8,vpad=8) [We use same for both].
        SSharp,        Default  1,   See MvTools MSuper(sharp=2)
        SRFilter,      Default  4,   See MvTools MSuper(rfilter=2)

                       MAnalyse args:
        ABlkSize,      Default 16,   See MvTools MAnalyse(BlkSize=8,BlkSizeV=8) [We use same for both].
        AOverLap,      Default  4,   See MvTools MAnalyse(OverLap=0)
        ASearch,       Default  3,   See MvTools MAnalyse(Search=4)
        ADct,          Default  0,   See MvTools MAnalyse(Dct=0). Using other than default value can be VERY slow.

                       MRecalculate args:
        RBlkSize,      Default   8,   See MvTools MRecalculate(BlkSize)
                         NOTE, RBlkSize = 0 Will Switch OFF MRecalculate, probably a bit faster at possible expense of quality.
        ROverlap,      Default   2,   See MvTools MRecalculate(Overlap)
        RthSAD,        Default 100,   See MvTools MRecalculate(thSAD=200)

                       MFlowInter args:
        Iml,           Default 200.0, See MvTools MFlowInter(ml=100.0)
        IBlend,        Default True,  See MvTools MFlowInter(Blend=True). Blend or copy frame at scene change.
        IthSCD1,       Default 400,   See MvTools MFlowInter(thSCD1=400) :  (0 < IthSCD1 <= (8*8)*255)
        IthSCD2,       Default 130,   See MvTools MFlowInter(thSCD1=130) :  (0 < IthSCD1 <= 255)
                       Threshold which sets how many blocks have to change for the frame to be considered as a
                       scene change. It is ranged from 0 to 255, 0 meaning 0 %, 255 meaning 100 %.
                       Default is 130 (which means 51.0%).
                       Used by MvTools2 MFlowInter during interpolation.

        Show,          Default True, Show Metrics.
        Verb,          Default True, If Verb And Show, then show additional metrics.

        SOSthSCD2      Default 130. As for MSCDetection(thSCD2=SOSthSCD2). Halt backwards search for JerkFrame where StartOfScene.

        Simple metrics on Show, "nnnn] UD : S : CIR", where nnnn is frame number, and character Flags highlited where,
                Duplicity2 Status:
            U = Unique frame status from Dupliicty DBase (non dupe, includes low motion static scene etc).
            D = Dupe frame status from Dupliicty DBase.

            :
            S = Current frame is Start Of Scene.

            :   DropDeadGorgeous Decisions :
            C = Copy frame (original as source frame)
            I = Interpolated frame
                Interpolated frames also show eg {I1/2:ILft=1275,IRgt=1276}
                    where, first of 2 Interpolated frames, and interpolate source frames are 1275 and 1276(JerkFrame usually, unless DupeS-1=SOS).
            R = Relocated frame from earlier in clip (interpolated frames shove following frames to later in clip, removing dupes)
                Reloacted frames also show eg {R1/8:Src=1276,n-2}
                    where, first of 8 relocated frames, source for current relocated frame is frame 1276, which is current frame-2.

            If Verb=True, also shows eg below
             Motion(n-1,n)=0.367            # Motion metric between current frame and current frame-1 (JerkFrame search metric).
             ScanMaxdist=9 (Cnt=7) @ 3458   # Maximum ScanAhead distance found to dupes (count of instances found), and most recent @ frame 3458.
             MostCommonScanDist=8 (Cnt=190) # The most common ScanAhead distance found between jerkframe and dupes, and instance count.
                                            # Maybe reduce ScanAhead arg to value as shown for MostCommonScanDist.

        Can use any Duplicity Mode in creating DBase, the Duplicity2 End+1 Scene change thing has no effect here (not necessary).

        It may be possible to up framerate to a non multiple of source whilst keeping all source frames verbatim, where eg source
        is 24 FPS and required = 25 FPS, and using,
        Src24.ChangeFPS(25) to produce a new clip at 25 FPS with an exact duplicate every 24 source frames and save new source as eg Src25.avi.
        Process new source with eg src25.Duplicity2(ThG=0.1,DBase="D:\Src25_Det.DB"), to produce a DBase (perhaps with MaxDupLen=1, perhaps not).
        Then use Src25.DropDeadGorgeous("D:\Src25_Det.DB",ScanAhead=12) to interpolate dupe frames achieving 25.0 and with all original
        source frames. With a ScanAhead of eg 12, will locate dupes and then backwards scan up to 12 frames before the dupes looking for
        frame of greatest movement, insert an interpolated frame prior to that greatest movement frame and shove all following frames along one
        to remove the dupe. Might be worth someone experimenting.
        Or, you could just use eg Duplicity2(Mode=2,ThG=0.1) to interpolate the exact dupes directly and without DropDeadGorgeous but that might
        produce a more rhythmic stutter every 25 frames.

*/

[StainlessS]

DropDeadGorgeous.avsi part 2

Function DropDeadGorgeous(clip c,String DB,Int "ScanAhead",
    \ Int "X",Int "Y",Int"W",Int "H",
    \ Int "SPad", Int "SSharp", Int "SRFilter",                         [* MSuper          *]
    \ Int "ABlkSize", Int "AOverlap", Int "ASearch",Int "ADct",         [* MAnalyse        *]
    \ Int "RBlkSize", Int "ROverlap", Int "RthSAD",                     [* MRecalculate    *]
    \ Float "Iml", Bool "IBlend", Int "IthSCD1", Int "IthSCD2",         [* MFlowInter      *]
    \ Bool "Show", Bool "Verb",Int "SOSthSCD2"
    \ ) {
    c   myName="DropDeadGorgeous: "     VER=2.13
    # RT_Version(), x.yyyzz where yyy==999, denotes version (x.yyy+0.001) Beta zz ie 1.99911 = v2.00 Beta 11.
    Assert(RT_Version() >= 1.99911,RT_String("%sNeed RT_Stats v2.00Beta11(%s)",myName,RT_VersionString))
    IsAvsPlus=(FindStr(UCase(versionString),"AVISYNTH+")!=0)    HasGScript=RT_FunctionExist("GScript")
    HasGRunt =RT_FunctionExist("GScriptClip")                   HasMvTools=RT_FunctionExist("MSuper")
    HasMaskTools=RT_FunctionExist("MT_Lutxy")                   HasRemoveGrain=RT_FunctionExist("RemoveGrain")
    HasCallCmd=RT_FunctionExist("CallCmd")
    ScanAhead = Default(ScanAhead,9)
    X=default(X,0)                      Y=default(Y,0)                      W=default(W,0)                    H=default(H,0)
    SPad      = Default(SPad,16)        SSharp  = Default(SSharp,1)         SRFilter= Default(SRFilter,4)
    ABlkSize  = Default(ABlkSize,16)    AOverlap= Default(AOverlap,4)       ASearch = Default(ASearch,3)      ADct   = Default(ADct,0)
    RBlkSize  = Default(RBlkSize,8)     ROverlap= Default(ROverlap,2)       RthSAD  = Default(RthSAD,100)
    Iml       = Default(Iml,200.0)      IBlend  = Default(IBlend,True)      IthSCD1 = Default(IthSCD1,400)    IthSCD2= Default(IthSCD2,130)
    Show      = Default(Show,True)      Verb=Show?Default(Verb,True):False  SOSthSCD2=Default(SOSthSCD2,130)
    Assert(0 < ScanAhead <= 60,RT_String("%s0 < ScanAhead(%d) <= 60",myName,ScanAhead))
    ####
    Assert(DB!="",RT_String("%sDB Cannot be ''",myName))
    DB1 = RT_GetFullPathName(DB)
    NameOnly=RT_FilenameSplit(DB,12) # Just name and extension parts, no Drive/Path
    Assert(RT_DBaseGetID(DB,0) == VER,RT_String("%s'%s' Wrong version DBase(v%.2f), require v%.2f",myName,NameOnly,RT_DBaseGetID(DB,0),VER))
    Assert(RT_DBaseGetID(DB,1) == c.Framecount,RT_String("%s'%s' DBase is incomplete",myName,NameOnly))
    W = (W > 0) ? (W + X) - Width  : W     H = (H > 0) ? (H + Y) - Height : H    # Make -ve relative width,height
    X = (X+3) / 4 * 4                      Y = (Y+3)  / 4 * 4                    # Round UP  (inwards) Mod 4
    W = (W-3) / 4 * 4                      H = (H-3)  / 4 * 4                    # Round DOWN(inwards) Mod 4
    W = Width - X + W                      H = Height - Y + H                    # +ve width,height
    DC=Crop(X,Y,W,H)
    FuncS="""
        Function Fn@@@(clip c,clip dc,String DB1,String DB2,int ScanAhead,Int SOSthSCD2,Bool Show,Bool Verb) {
            c    n=current_frame
            Stat2 = RT_DBaseGetField(DB2,n,0)
            if(Stat2 == 0) {
                Stat1 = RT_DBaseGetField(DB1,n,FLD_STAT@@@)
                if(Stat1 >= FLG_DE@@@) { # Is a dupe (user musta jumped about), do backwards scan
                    Index=RT_DBaseGetField(DB1,n,FLD_INDEX@@@)
                    DupS=n-Index                                                # Start of duplicate run
                    Count=RT_DBaseGetField(DB1,DupS,FLD_COUNT@@@)               # Dup Count stored only in first Record of Sub Sequence
                    DupE=DupS+Count-1                                           # End of dupe run
                    if(DupS < 2) { # Not enough prev frames to interpolate
                        for(i=0,DupE) { RT_DBaseSet(DB2,i,1,0,0,0) }            # Copy from 0, cant do anything else.
                    } Else {
                        ScanS = Max(DupS - ScanAhead, 1)
                        for(i=DupS-1,ScanS,-1) {                                # Make sure no dupes in backwards scan range
                            if(RT_DBaseGetField(DB1,i,FLD_STAT@@@) >= FLG_DE@@@) { ScanS=i+1 i=-1 }
                        }
                        mxi   = -1
                        mxDif = -1.0
                        for(i=DupS-1,ScanS,-1) {
                            if(SOS@@@.RT_AverageLuma(n=i).Int!=0) { i=-1 }      # Break at Start Of Scene with current max
                            Else {
                                dif = MFPC@@@.RT_AverageLuma(n=i)
                                if(dif>mxDif) { mxDif=dif mxi = i }
                            }
                        }
                        if(mxi<0) { # No JerkFrame found, ie DupS-1 is StartOfScene
                            if(DupE+1==c.FrameCount || SOS@@@.RT_AverageLuma(n=DupE+1).Int!=0) {
                                for(i=ScanS,DupE) { RT_DBaseSet(DB2,i,1,0,0,0) }               # Copy all, Cannot Interpolate.
                            } Else { # DupS-1 is SOS, and DupE+1 is NOT SOS, Interpolate dupes using source DupS-1 and DupE+1
                                for(i=ScanS,DupS-1) { RT_DBaseSet(DB2,i, 1,0,0,0) }            # Copy frames up to SOS frame
                                for(i=DupS,DupE)    { RT_DBaseSet(DB2,i, 2,COUNT,i-DupS,1) }   # Interp, Type SOS
                            }
                        } Else {
                            for(i=mxi,mxi+Count-1) { RT_DBaseSet(DB2,i, 2,COUNT,i-mxi,0) }     # Interpolate
                            Cnt2=DupE-(mxi+Count)+1
                            for(i=mxi+Count,DupE)  { RT_DBaseSet(DB2,i, 3,COUNT,i-mxi,Cnt2) }  # Relocate Frame relative (frame n = frame n - COUNT)
                            if(Verb) {
                                scandist = DupS - mxi
                                cnt = RT_DBaseGetAttrib(DB2,scandist) + 1
                                RT_DBaseSetAttrib(DB2,scandist,cnt)
                                if(cnt >= ScanComCnt@@@) {
                                    if(cnt == ScanComCnt@@@) {
                                        Global ScanComDist@@@=Max(ScanComDist@@@,scandist)
                                    } Else {
                                        Global ScanComCnt@@@  = cnt
                                        Global ScanComDist@@@ = scandist
                                    }
                                }
                                if(scandist>=ScanMaxDist@@@) {
                                    if(scandist==ScanMaxDist@@@) {
                                        Global ScanMaxPos@@@ = Max(mxi,ScanMaxPos@@@)
                                    } Else {
                                        Global ScanMaxDist@@@=scandist
                                        Global ScanMaxPos@@@=mxi
                                    }
                                }
                            }
                        }
                    }
                    # Frame Following this dupe run is a NON DUPE.
                } Else { # Non dupe
                    ScanE = Min(n+(ScanAhead*2),c.FrameCount-1)
                    DupS = -1
                    for(i=n+1,ScanE) { if(RT_DBaseGetField(DB1,i,FLD_STAT@@@) >= FLG_DE@@@) { DupS = i i=ScanE } } # find 1st dupe
                    if(DupS<0) { # dupe not found
                        for(i=n,Max(ScanE-ScanAhead,n)) { RT_DBaseSet(DB2,i,1,0,0,0) }   # Copy
                    } Else {
                        Count=RT_DBaseGetField(DB1,DupS,FLD_COUNT@@@)           # Dup Count stored only in first Record of Sub Sequence
                        DupE=DupS+Count-1
                        ScanS = Max(DupS - ScanAhead, n)
                        mxi   = -1
                        mxDif = -1.0
                        for(i=DupS-1,ScanS,-1) {                                # Search for JerkFrame
                            if(SOS@@@.RT_AverageLuma(n=i).Int!=0) { i=-1 }      # Break @ Start Of Scene with current max
                            Else {
                                dif = MFPC@@@.RT_AverageLuma(n=i)
                                if(dif>mxDif) { mxDif=dif mxi = i }
                            }
                        }
                        if(mxi<0) { # No JerkFrame found, ie DupS-1 is StartOfScene
                            if(DupE+1==c.FrameCount || SOS@@@.RT_AverageLuma(n=DupE+1).Int!=0) {
                                for(i=n,DupE) { RT_DBaseSet(DB2,i,1,0,0,0) }    # Copy all, Cannot Interpolate.
                            } Else { # DupS-1 is SOS, and DupE+1 is NOT SOS, Interpolate dupes using source DupS-1 and DupE+1
                                for(i=n,DupS-1)  { RT_DBaseSet(DB2,i, 1,0,0,0) }                # Copy frames up to SOS frame
                                for(i=DupS,DupE) { RT_DBaseSet(DB2,i, 2,COUNT,i-DupS,1) }       # Interp, Type=SOS
                            }
                        } Else {
                            for(i=n,mxi-1)          { RT_DBaseSet(DB2,i,1,0,0,0) }          # Copy frames prior to frame of greatest movement
                            for(i=mxi,mxi+Count-1)  { RT_DBaseSet(DB2,i, 2,COUNT,i-mxi,0) } # Interp
                            Cnt2=DupE-(mxi+Count)+1
                            for(i=mxi+Count,DupE)   { RT_DBaseSet(DB2,i, 3,COUNT,i-mxi,Cnt2) } # relocate frames
                            if(Verb) {
                                scandist = DupS - mxi
                                cnt = RT_DBaseGetAttrib(DB2,scandist) + 1
                                RT_DBaseSetAttrib(DB2,scandist,cnt)
                                if(cnt >= ScanComCnt@@@) {
                                    if(cnt == ScanComCnt@@@) {
                                        Global ScanComDist@@@=Max(ScanComDist@@@,scandist)
                                    } Else {
                                        Global ScanComCnt@@@  = cnt
                                        Global ScanComDist@@@ = scandist
                                    }
                                }
                                if(scandist>=ScanMaxDist@@@) {
                                    if(scandist==ScanMaxDist@@@) {
                                        Global ScanMaxPos@@@ = Max(mxi,ScanMaxPos@@@)
                                    } Else {
                                        Global ScanMaxDist@@@=scandist
                                        Global ScanMaxPos@@@=mxi
                                    }
                                }
                            }
                        }
                    }
                }
                Stat2 = RT_DBaseGetField(DB2,n,0)
            }
            if(Stat2==2)  { # Interpolated
                Count = RT_DBaseGetField(DB2,n,1)
                Index = RT_DBaseGetField(DB2,n,2)
                iType = RT_DBaseGetField(DB2,n,3)   # 0) Normal Interpolate with shove duplicates, 1) DupS-1=SOS
                if(iType==0) {
                    TmpC=c
                    FixFrm=n-Index-1
                } Else {
                    DupS=n-Index        DupE=DupS+Count-1
                    IFrms=3
                    LS=Max(DupS-IFrms,0)
                    LCnt=DupS-LS        RCnt=Min(c.FrameCount-1-DupE,IFrms)
                    TmpC = c.Trim(LS,-LCnt) ++ c.Trim(DupE+1,-RCnt)
                    FixFrm=LCnt-1
                }
                Prefilt=TmpC.RemoveGrain(22)
                Pad=SPad@@@ Sharp=SSharp@@@ srfilter=SRFilter@@@ Ablksize=ABlkSize@@@ AOverLap=AOverLap@@@ ASearch=ASearch@@@ ADct=ADct@@@
                Super=TmpC.MSuper(hpad=Pad,vpad=Pad,levels=1,sharp=Sharp,rfilter=srfilter)
                Superfilt=Prefilt.MSuper(hpad=Pad,vpad=Pad,sharp=Sharp,rfilter=srfilter)
                bv=Superfilt.MAnalyse(isb=true  ,blksize=Ablksize,overlap=AOverLap,search=ASearch,delta=1,dct=ADct)
                fv=Superfilt.MAnalyse(isb=False ,blksize=Ablksize,overlap=AOverLap,search=ASearch,delta=1,dct=ADct)
                if(RBlkSize@@@>0) {
                    RBlkSize=RBlkSize@@@ ROverLap=ROverLap@@@ RthSAD=RthSAD@@@
                    bv=Super.MRecalculate(bv,blksize=RBlkSize,overlap=ROverLap,thSAD=RthSAD)
                    fv=Super.MRecalculate(fv,blksize=RBlkSize,overlap=ROverLap,thSAD=RthSAD)
                }
                TmpC=TmpC.MFlowInter(Super,bv,fv,time=100.0*(Index+1)/(Count+1),ml=Iml@@@,blend=IBlend@@@,thSCD1=IthSCD1@@@,thSCD2=IthSCD2@@@)
                TmpC.Trim(FixFrm,-1)
                if(Show) {
                    DStat=(RT_DBaseGetField(DB1,n,0)>=FLG_DE@@@)
                    sos = (SOS@@@.RT_AverageLuma(n=n).Int!=0)
                    if(iType==0)    { iLft=FixFrm iRgt=iLft+1 }
                    else            { iLft=DupS-1 iRgt=DupE+1 }
                    if(Verb) {
                        mot = MFPC@@@.RT_AverageLuma(n=n)
                        ScanMaxDist=ScanMaxDist@@@
                        ScanMaxDistCnt=RT_DBaseGetAttrib(DB2,ScanMaxDist)
                        RT_Subtitle(Fmtv_I@@@,n,DStat?76:45,DStat?45:76,sos?45:76,Index+1,Count,ILft,IRgt,
                            \ mot,ScanMaxDist,ScanMaxDistCnt,ScanMaxPos@@@,ScanComDist@@@,ScanComCnt@@@)
                    } Else { RT_Subtitle(Fmt_I@@@,n,DStat?76:45,DStat?45:76,sos?45:76,Index+1,Count,ILft,IRgt) }
                }
            } Else if(Stat2==3) { # Relocated
                DStat=(RT_DBaseGetField(DB1,n,0)>=FLG_DE@@@)
                RelocOffset = RT_DBaseGetField(DB2,n,1)
                c.Trim(n-RelocOffset,-1)  # Relocated, n - count
                if(Show) {
                    Count = RT_DBaseGetField(DB2,n,3)
                    Index = RT_DBaseGetField(DB2,n,2) - RelocOffset
                    sos = (SOS@@@.RT_AverageLuma(n=n).Int!=0)
                    if(Verb) {
                        mot = MFPC@@@.RT_AverageLuma(n=n)
                        ScanMaxDist=ScanMaxDist@@@
                        ScanMaxDistCnt=RT_DBaseGetAttrib(DB2,ScanMaxDist)
                        RT_Subtitle(Fmtv_R@@@,n,DStat?76:45,DStat?45:76,sos?45:76,Index+1,Count,n-RelocOffset,RelocOffset,
                            \ mot,ScanMaxDist,ScanMaxDistCnt,ScanMaxPos@@@,ScanComDist@@@,ScanComCnt@@@)
                    } Else { RT_Subtitle(Fmt_R@@@,n,DStat?76:45,DStat?45:76,sos?45:76,Index+1,Count,n-RelocOffset,RelocOffset) } }
            } Else if(Show) { # Copy original
                DStat=(RT_DBaseGetField(DB1,n,0)>=FLG_DE@@@)
                sos = (SOS@@@.RT_AverageLuma(n=n).Int!=0)
                if(Verb) {
                    mot = MFPC@@@.RT_AverageLuma(n=n)
                    ScanMaxDist=ScanMaxDist@@@
                    ScanMaxDistCnt=RT_DBaseGetAttrib(DB2,ScanMaxDist)
                    RT_Subtitle(Fmtv_C@@@,n,DStat?76:45,DStat?45:76,sos?45:76,
                        \ mot,ScanMaxDist,ScanMaxDistCnt,ScanMaxPos@@@,ScanComDist@@@,ScanComCnt@@@)
                } Else { RT_Subtitle(Fmt_C@@@,n,DStat?76:45,DStat?45:76,sos?45:76) }
            }
            Return Last
        }

[StainlessS]

DropDeadGorgeous.avsi part 3

        Function StartOfSceneClip(clip c,Int "thSCD1",Int "thSCD2") { # All Luma Samples set 255 at SOS, else 0
            thSCD1=Default(thSCD1,400)  thSCD2=Default(thSCD2,130)
            sup=c.MSuper(pel=1,sharp=0,rfilter=2,hpad=16, vpad=16)
            fv=sup.MAnalyse(isb=false,delta=1,blksize=16)
            Return c.MSCDetection(fv,thSCD1=thSCD1,thSCD2=thSCD2)
        }
        Function MMaskFromPrevClip(clip c,Int "MaskType",Float "Gamma",Int "thSCD1",Int "thSCD2") {
            # MaskType:- 0=Motion, 1=SAD, 2=Occlusion, 3=Horizontal, 4=Vertical, 5=ColorMap.
            MaskType=Default(MaskType,0)  Gamma=Default(Gamma,1.0)  thSCD1=Default(thSCD1,400)  thSCD2=Default(thSCD2,130)
            sup=c.MSuper(pel=1,sharp=0,rfilter=2,hpad=16, vpad=16)
            fv=sup.MAnalyse(isb=false,delta=1,blksize=16)
            Return c.MMask(fv,Gamma=1.0,kind=MaskType,thSCD1=thSCD1,thSCD2=thSCD2)
        }
        DB2 = RT_GetFullPathName("~@@@_"+RT_LocalTimeString+".DB")    RT_DBaseAlloc(DB2,FrameCount,"iiii")
        Global ScanComDist@@@=0    Global ScanComCnt@@@=0
        Global ScanMaxDist@@@=0    Global ScanMaxPos@@@=0
        Global SOS@@@ = dc.StartOfSceneClip(thSCD2=SOSthSCD2)
        Global MFPC@@@= dc.MMaskFromPrevClip(Gamma=1.0,MaskType=0,thSCD1=8*8*255,thSCD2=255) # Motion, Switched OFF SC detection, we want all results.
        ### Some DB1 Fields:-
        Global FLD_STAT@@@   = 0   # Holds Status FLGS
        Global FLD_INDEX@@@  = 4   # Index into Count (0 Relative, 0->Count-1)
        Global FLD_COUNT@@@  = 5   # Sequence Frame Count
        ### Some DB1 Status Flags in Field FLD_STAT@@@
        Global FLG_DE@@@     = 7   # Duplicate Sequence is followed by Scene Change (below is non dupe, greater or equalis dupe)
        ###
        Global SPad@@@=SPad           Global SSharp@@@=SSharp       Global SRFilter@@@=SRFilter
        Global ABlkSize@@@=ABlkSize   Global AOverLap@@@=AOverLap   Global ASearch@@@=ASearch        Global ADct@@@=ADct
        Global RBlkSize@@@=RBlkSize   Global ROverLap@@@=ROverLap   Global RthSAD@@@=RthSAD
        Global Iml@@@=Iml             Global IBlend@@@=IBlend       Global IthSCD1@@@=IthSCD1        Global IthSCD2@@@=IthSCD2
        Global Fmt_C@@@ ="%d] \a%cU\a%cD\a- : \a%cS\a- : C\aLIR"
        Global Fmt_I@@@ ="%d] \a%cU\a%cD\a- : \a%cS\a- : \aLC\a-I\aLR\a- {I%d/%d:Lft=%d,IRgt=%d}"
        Global Fmt_R@@@ ="%d] \a%cU\a%cD\a- : \a%cS\a- : \aLCI\a-R\a- {R%d/%d:Src=%d,n-%d}"
        Global Fmtv_C@@@=Fmt_C@@@ + "\a-\nMotion(n-1,n)=%.3f\nScanMaxDist=%d (Cnt=%d) @ %d\nMostCommonScanDist=%d (Cnt=%d)"
        Global Fmtv_I@@@=Fmt_I@@@ + "\a-\nMotion(n-1,n)=%.3f\nScanMaxDist=%d (Cnt=%d) @ %d\nMostCommonScanDist=%d (Cnt=%d)"
        Global Fmtv_R@@@=Fmt_R@@@ + "\a-\nMotion(n-1,n)=%.3f\nScanMaxDist=%d (Cnt=%d) @ %d\nMostCommonScanDist=%d (Cnt=%d)"
        ###
        ARGS = "dc,DB1,DB2,ScanAhead,SOSthSCD2,Show,Verb"
        ScriptLine = "Fn@@@("+ARGS+")"
        Last.GScriptClip(ScriptLine, local=true, args=ARGS)
    """
    GIFunc="DROPDEADGORGEOUS"          # Function Name, Supply unique name for your multi-instance function.
    GIName=GIFunc+"_InstanceNumber"    # Name of the Instance number Global
    RT_IncrGlobal(GIName)              # Increment Instance Global (init to 1 if not already exists)
    GID   = GIFunc + "_" + String(Eval(GIName))
    InstS = RT_StrReplace(FuncS,"@@@","_"+GID)
    # Write Debug GScript File to current directory (line numbers will correspond to error messages)
#   RT_WriteFile("DEBUG_"+GID+".TXT","%s",InstS)
    HasGScript ? GScript(InstS) : Eval(InstS,"GS_EVAL") # Use GSCript if installed (loaded plugs override builtin)
    # if CallCmd available, Auto delete DBase file on clip closure.
    HasCallCmd?CallCmd(close=RT_String("""CMD /C chcp 1252 && del "%s" """,DB2), hide=true, Synchronous=7):NOP
    Return Last
}

[StainlessS]

From post #46,

It may be possible to up framerate to a non multiple of source whilst keeping all source frames verbatim, where eg source
is 24 FPS and required = 25 FPS, and using,
Src24.ChangeFPS(25) to produce a new clip at 25 FPS with an exact duplicate every 24 source frames and save new source as eg Src25.avi.
Process new source with eg src25.Duplicity2(ThG=0.1,DBase="D:\Src25_Det.DB"), to produce a DBase (perhaps with MaxDupLen=1, perhaps not).
Then use Src25.DropDeadGorgeous("D:\Src25_Det.DB",ScanAhead=12) to interpolate dupe frames achieving 25.0 and with all original
source frames. With a ScanAhead of eg 12, will locate dupes and then backwards scan up to 12 frames before the dupes looking for
frame of greatest movement, insert an interpolated frame prior to that greatest movement frame and shove all following frames along one
to remove the dupe. Might be worth someone experimenting.

I tried above converting [EDIT: true] 25.0 FPS to 30.0FPS, using ChangeFPS(30.0), and Duplicity2(ThG=0.1,MaxDuplen=1),
Then, used DropDeadGorgeous(ScanAhead=5), result produced is 30FPS with all original source frames, and with good fluid motion throughout.
ChangeFPS(30.0) inserts duplicate after every five source frames, and DropDeadGorgeous(ScanAhead=5) introduces
an interpolated frame within 5 frames prior to the duplicate (between frames of greatest motion) and pushes the following frames one position
along to squeeze out the duplicate.
Works pretty good.
EDIT: If source clip was badly decimated from eg 30FPS to 25 and with jerky motion due to decimation, above should produce pretty much the best possible result and repair/tween the jerk frames produced by the decimation.
EDIT: Should also perform well in cases where cadence changes, eg where source was decimated from 29.97 FPS to 25 FPS,
or where source was edited after decimation, neither one should be any problem.

[Frank62]

A few months ago this could have saved my life...
I have some sources to test - when I will have time (can last longer) - I will tell you the result!

[StainlessS]

Good, thank you, been hoping for some kind of feedback.

[Puffy2010]

Hi, I am unable to download RT_Stats_25&26_x86_x64_dll_v2.00Beta12_20181125.zip from your mediafire. I have tried importing it to my mediafire and it still gets stuck in a loop requesting new download keys. I can download all the other files on your mediafire but this one seems broken.

I would really like to give Duplicity2 a whirl

Can you please re-upload it?

[ChaosKing]

Yeah download_repair.php does not sound good. It's an endless loop

Why don't you switch to dropbox or something similar (you know github ). The biggest benefit for you would be: The links doesn't change if you update the file in dropbox. I think what was one of your complaints about mediafire..

[StainlessS]

Download failed for me also, was new problem, dont know what happend.
Have re-compressed as 7z instead of zip, and re-upped to both MediaFire and SendSpace, Mediafire downed ok.
Strange, failed down had weird interface that I aint seen before, anyways, working now.

Github, one day.
Am about to down VSCodian [or whatever the pure open source version of Visual Studio Code is called], so github almost at my fingertips.

[TheUpsideDown]

Hi StainlessS,

Thanks for all your tools! I've been making my way through some of them and the functionalities they offer are astounding!

We occasionally get get material in that exhibits skipped--and then duplicated frames. We don't have to correct the material--we just have to flag that there are duplicates and inform the distributor.

I've been using Duplicity and the output of frames.txt, pasted into excel with a formula to return non-sequential frame numbers to spot duplicates.

I'm wondering if there's a way to "reverse" the output of frames.txt so that it only writes the frame numbers of duplicate frames. Or, perhaps a way to accomplish this that I'm not gleaning from the notes. Or is there another better suited to this task (I've only dabbled with AVIsynth over the years).

Thanks for any help you can provide!

[StainlessS]

Maybe tommrroew,
I'll tyr, but at the momomentt, I a bit less that totally cohernet. Maybe when I'm lees overwhelmed.

[StainlessS]

OK, I'm assuming that you only want single frames written and not ranges (although this threw me a little bit ""reverse" the output of frames.txt").

first few lines of the client script [Added in BLUE]

Import(".\Duplicity2.Avsi")

######
FN  = ".\Earth.AVI"        DBase  = "D:\Earth.DB"            ThB=2.0
#FN  = "D:\G.AVI"          DBase  = "D:\G_Det.DB"            ThB=2.0
#FN  = "D:\GTEST.AVI"      DBase  = "D:\GTEST_Det.DB"        ThB=2.0
#FN = "D:\HAR.AVI"         DBase  = "D:\HAR_Det.DB"          ThB=2.5
#FN = "D:\DeDupTest.AVI"   DBase  = "D:\DeDupTest_Det.DB"    ThB=2.0
######
Avisource(FN)
SRC=Last
###
Mode=0             # 0=Write Frames only : 1=Replace Dupes with Exact Dupes : 2= Interpolate/Blend Duplicates.
ThG=1.0            # Primary Frame Global Detect
ThP=64             # Pixel Count Detect
MaxDupLen=9        # Max detected, if more then is considered static sequence not duplicate)
MaxInterp=9        # Max Interpolated frames when Mode=2 (More than MaxInterp & less or equal to MaxDupLen then uses Blend mode)
InterpFast=true
Ranges=False    # Added
Show=True
ShowDot=False
ShowBlk=true
VERB=False
###
TITLE=False
STACK=True
STACK4=False
###

Result=SRC.Duplicity2(Mode=Mode,ThG=ThG,ThB=ThB,ThP=ThP,MaxDupLen=MaxDupLen,MaxInterp=MaxInterp,
    \ Show=False,InterpFast=InterpFast,Ranges=RANGES,DBase=(STACK4||!SHOW)?DBase:"")

Metrics=SRC.Duplicity2(Mode=Mode,ThG=ThG,ThB=ThB,ThP=ThP,MaxDupLen=MaxDupLen,MaxInterp=MaxInterp,
    \ Show=True,Verb=VERB,ShowBlk=ShowBlk,ShowDot=ShowDot,InterpFast=InterpFast,Ranges=RANGES,DBase=(STACK4||!SHOW)?"":DBase)

        Frames,       Default "Frames.txt", Frames filename for output, ("" will not write Frames file).
        Ranges,       Default True (True=Write comma separated frame ranges to Frames file, else single frames only).
        FrameAsRange, Default False. If Ranges=True and FrameAsRange=default false, and single frame, then writes single frame number and
                      not range. If Ranges=True and FrameAsRange=True then writes single frame as range eg '23,23' for frame 23.

So this

###
# Duplicate frames file. [ Generated by Duplicity2() ]
###

90,92
320
382
384
386
388,389
439
490
492
527
553
567
583
617
628
635
639
650
660
669
673
682
684
686
688,689

would become this

###
# Duplicate frames file. [ Generated by Duplicity2() ]
###

90
91
92
320
382
384
386
388
389
439
490
492
527
553
567
583
617
628
635
639
650
660
669
673
682
684
686
688
689

Perhaps I have misinterpreted your request, if so then please restate.

[StainlessS]

And this would convert Frames.txt to a list of only non duplicate frames, requires FrameSel():- https://forum.doom9.org/showthread.php?t=167971

AviSource(".\Earth.avi")
trim(0,-10000)  # Trim exact same as Duplicity2 source clip

InFRAMES=".\Frames.txt"
OutFRAMES=".\OutFrames.txt"
REJECT=True  # output True=NON Dupes : False = Dupes [but as for below RANGES]
RANGES=False # single frames not ranges

FrameSel_CmdReWrite(OutFrames,Cmd=InFRAMES,Reject=REJECT,Ordered=True,Range=RANGES,Prune=False)

Return Last

Note, you do not have to play the clip to generate the outFrames.txt file, is created before first frame is produced.

EDIT: And welcome to the forum

EDIT: And to repair dupes, might like to try DropDeadGorgeous2() with SCANAHEAD = (Dupe<===>Dupe Distance - 1) with all original non-dupe frames kept in output [SCANAHEAD max 60].

EDIT: Added config REJECT=True, output non dupes, but can change to false to just change to output single frames or ranges as for RANGES setting,
so can output either dupes or non dupes, and ranges or single frames.

[TheUpsideDown]

Originally Posted by StainlessS

Note, you do not have to play the clip to generate the outFrames.txt file, is created before first frame is produced.

EDIT: And welcome to the forum

EDIT: And to repair dupes, might like to try DropDeadGorgeous2() with SCANAHEAD = (Dupe<===>Dupe Distance - 1) with all original non-dupe frames kept in output [SCANAHEAD max 60].

EDIT: Added config REJECT=True, output non dupes, but can change to false to just change to output single frames or ranges as for RANGES setting,
so can output either dupes or non dupes, and ranges or single frames.

Thanks for the warm welcome!
Apologies for the late reply--and it seems I have a confession to make:

I had originally tried using Duplicity2 to flag repeated frames as it seems that is an easily exposed function (mode=0) but ran into issues with setup I couldn't overcome.*

I was successful in using ExactDeDup--this is what my question was actually referencing--sorry! I can get ExactDeDup to write an output file of all the frames that are unique, but couldn't figure out how to replicate the idea of only writing the duplicated frames.


* my issue(s)
I installed base AVISynth and then UniversalAvisynthInstaller to allow jumping between multiple forks/versions.
I made sure to install the files referenced in post #1:

RT_Stats v2.00_Beta_11 + CallCmd
GSCript, Grunt
MaskTools2, MvTools2
RemoveGrain

Calling Duplicity2() results in:

RT_DBaseAlloc: Cannot Query Filesystem
([GScript], line 260)
(C:/Users/user/UniversalAvisynthInstaller/AVSPLUS_x86/plugins/Duplicity2.Avsi, line 760)
(New File (1), line 9)

Normally I'm a half-ways intelligent guy and googling the errors I receive (in other scenarios for instance) leads me to certain dependencies I've overlooked. Sadly I can't find much to go on with the above.

If this is something obvious that a noob should know, I'd love to get pointed in the right direction...

[StainlessS]

I've never seen that error message before, can only assume that something is preventing successfull call to QueryFatVolume(fn). [In BLUE in code block]

AVSValue __cdecl RT_DBaseAlloc(AVSValue args, void* user_data, IScriptEnvironment* env) {	
	const char * myName	= "RT_DBaseAlloc: ";
	const char * fn		= args[0].AsString();
	const int records	= args[1].AsInt();
	const char *typestr	= args[2].AsString();
	const int StringlenMax = args[3].AsInt(256);
	if(*fn=='\0')	env->ThrowError("%sEmpty Filename",myName);

	__int64 MaxFileSz = 0xFFFFF00000i64 ;												// 1TB - 1MB

	int fatvol = QueryFatVolume(fn);
	if(fatvol < 0)	env->ThrowError("%sCannot query Filesystem",myName);

	__int64 dfs = QueryDiskFreeSpace(fn) - 0x100000;									// minus 1MB
	if(dfs < 0)	env->ThrowError("%sCannot query DiskFreeSpace",myName);
	// dprintf("DiskFreeSpace=$%I64X",dfs);
	__int64 maxfs = (fatvol==1)? 0xFFF00000i64 : MaxFileSz; 	// limit 4GB on FAT32
	__int64 maxcurdfs=min(maxfs,dfs);							// Max current space, Limit to free space available to user.

I'm wondering if the RT_stats.dll is being partially blocked.
Right click on RT_Stats.dll and look at Properties, does it say something like
"This file came from another computer and has been blocked for your protection".


Just curious, what OS are you using ?

Here AlternateStreamView, to view and delete alternate streams, (Zone Identifier is the blocking thing):- https://www.nirsoft.net/utils/altern...a_streams.html

AlternateStreamView is a small utility that allows you to scan your NTFS drive, and find all hidden alternate streams stored in the file system. After scanning and finding the alternate streams, you can extract these streams into the specified folder, delete unwanted streams, or save the streams list into text/html/csv/xml file.

And from another post

Originally Posted by StainlessS

For plugin problems where dll's have been Blocked when downloading from the net (or any other files eg pdf etc).
(Problem dll's show in file Properties as BLOCKED, as mentioned two posts earlier).

Streams.exe by SysInternals (Micro$oft) for removing alternatve data streams:-
https://docs.microsoft.com/en-us/sys...nloads/streams
Docs say Vista+, but seems to work just fine on XP.

And below referring to above as post #18

Originally Posted by StainlessS

Further to previous posts on blocked dlls (or blocked any type files),
If you need to strip the alternate stream data from any files, and do not have net access to download Streams.exe from SysInternals (Microsoft),
then can just copy files to a FAT32 partition, (as FAT32 dont have alternate streams), and then just copy back and overwrite original files on your
NTFS partition.

EDIT: Link to Streams.Exe @ SysInternals in post #18. (XP+, despite web page saying Vista+).

EDIT: Better streams Alternate Streams Viewer, can delete Zone Identifier Alternate data stream.
https://www.nirsoft.net/utils/altern...a_streams.html

EDIT:
OOps, I thought QueryFatVolume was a system call, its actually one of mine, as here

int __cdecl QueryFatVolume(const char *relname) {
	// Return:- 1=FAT. 0 = Not FAT. -1 on error;
	int ret = -1;
	char FullPath[_MAX_PATH];
	if(_fullpath(FullPath, relname, _MAX_PATH ) != NULL ) {
		TCHAR RootPathName[_MAX_PATH];
		_splitpath(FullPath,RootPathName, NULL,NULL,NULL );
		char *p=RootPathName;
		while(*p++);
		--p;
		if(p>RootPathName && p[-1] != '\\') {*p++='\\';*p='\0';}
		TCHAR FileSystemNameBuffer[MAX_PATH+1];
		BOOL result = GetVolumeInformation(RootPathName,NULL,0,NULL,NULL,NULL,FileSystemNameBuffer,MAX_PATH+1);
		if(result) {
			ret = (_strnicmp(FileSystemNameBuffer,"FAT",3)==0) ?1:0;		// Just the 1st 3 characters (FAT/FAT32)
		}
	}
	return ret;
}

// # Above, I seem to have mixed _MAX_PATH and MAX_PATH, but they equate to the same thing so should not be source of problem

I'm wondering, how long is your filename ?
Show filename path you are using, if relative path then ALSO give full path.
Also, what is fileSystem, ExtFat ?

EDIT: I Suggest scanning your plugins directories (or better still the UniversalAvisynthInstaller directory) with AlternateStreamView, and delete any Zone Identifier streams.

EDIT: A bit more on preventing blocked files using GPEdit.msc (Pro versions OS only):- https://forum.doom9.org/showthread.p...09#post1872609

EDIT: Also look at properties of your clip, dont know if video clips are ever blocked. [I think but am not sure, that I have seen clips blocked]