StainlessS
7th August 2016, 20:12
SOLUTION:
Are you monitoring temperatures to check for thermal throttling?
Hi guys,
Getting some strange results where machine (Core Duo 2.4GHz, Duel Core, XP32SP3) slows over time doing exact same scans on YV12 ColorBars of duration 4 Hours.
Here Complete source to qCombed() v1.00 plugin:- (D9 thread http://forum.doom9.org/showthread.php?t=173754)
Only mods to published source are concerned with timing and DebugView output (mods marked in blue).
/*
qCombed() [ IsCombed() Mod by StainlessS @ Doom9.org.]
Mod of some of the source code in DeComb filter, FieldDeinterlace.cpp.
Original Decomb dll and source (c) Donald Graft, can be found here:- http://rationalqm.us/mine.html
*/
/*
qCombed() v1.0, 06 Aug 2016. First Release.
*/
#include <windows.h>
#include <stdio.h>
#include <time.h> // ssS Added for timings
#include "avisynth.h" // VERSION 3 header
// ssS Added
int __cdecl dprintf(char* fmt, ...) {
char printString[2048]="qCombed: ";
char *p=printString;
for(;*p++;);
--p; // @ null term
va_list argp;
va_start(argp, fmt);
vsprintf(p, fmt, argp);
va_end(argp);
for(;*p++;);
--p; // @ null term
if(printString == p || p[-1] != '\n') {
p[0]='\n'; // append n/l if not there already
p[1]='\0';
}
OutputDebugString(printString);
return p-printString; // strlen printString
}
AVSValue __cdecl GetVar(IScriptEnvironment* env, const char* name) {
try {return env->GetVar(name);} catch (IScriptEnvironment::NotFound) {} return AVSValue();
}
// Uses reference to dst else error (same dst for entire clip scan).
bool __cdecl qCombed_Planar(PVideoFrame src,PVideoFrame &dst,const int n,const int Th,int *counts,const int boxh,const int boxw) {
const BYTE *srcp = src->GetReadPtr(PLANAR_Y);
const int pitch = src->GetPitch(PLANAR_Y);
const w = src->GetRowSize(PLANAR_Y);
const h = src->GetHeight(PLANAR_Y);
// use box size of x=4 by y=8
const int boxarraysize = boxh * boxw;
const int CT = 4 * 8 / 2 - 1; // 15, (greater than this is at least half of max possible)
memset(counts, 0, sizeof(counts[0])*boxarraysize);
BYTE *dstp = dst->GetWritePtr(PLANAR_Y);
const int dpitch = dst->GetPitch(PLANAR_Y);
const dh = dst->GetHeight(PLANAR_Y);
const BYTE *above = srcp - pitch;
const BYTE *below = srcp + pitch;
memset(dstp,0,dh*dpitch); // dst is NewVideoFrame, ie not cropped.
int x, y;
for (y = 1; y < h - 1; ++y) {
srcp += pitch;
above += pitch;
below += pitch;
dstp += dpitch;
for (x = w; --x>=0;) {
const int val = ((long)above[x] - (long)srcp[x]) * ((long)below[x] - (long)srcp[x]);
if(val > Th) dstp[x] = 0xFF;
}
}
const BYTE *dp = dst->GetReadPtr(PLANAR_Y);
above = dp - dpitch;
below = dp + dpitch;
for (y = 1; y < h - 1; y++) {
const int boxy = (y >> 3) * boxw;
dp += dpitch;
above += dpitch;
below += dpitch;
for (x = w; --x>=0;) {
if (dp[x] == 0xFF && above[x] == 0xFF && below[x] == 0xFF) {
++counts[boxy + (x >> 2)];
}
}
}
bool ret=false;
for (x = boxarraysize; --x>=0;) {
if (counts[x] > CT) {ret = true; break;}
}
return ret;
}
AVSValue __cdecl qCombed(AVSValue args, void* user_data, IScriptEnvironment* env) {
char *myName="qCombed: ";
if(!args[0].IsClip()) env->ThrowError("%sMust have a clip",myName);
PClip child = args[0].AsClip();
const VideoInfo &vi=child->GetVideoInfo();
const bool yuy2 = vi.IsYUY2();
if(!vi.IsPlanar() && !yuy2) env->ThrowError("%sPlanar or YUY2 Only",myName);
if(vi.width==0||vi.num_frames<=0) env->ThrowError("%sClip has no video",myName);
int Th=args[1].AsInt(20);
if(Th<0 || Th>255) env->ThrowError("%sThreshold 0 -> 255",myName);
Th *= Th; // Squared
FILE *fp=NULL;
const fc=vi.num_frames;
int n=0;
const char *FileName = args[2].AsString("");
bool fpmd=(*FileName!='\0');
char ebf[256]=""; // No error so far
AVSValue ret;
if (!fpmd){ // Conditional filter function
if(args[6].IsInt()) {n = args[6].AsInt(); } // Frame n
else {
AVSValue cn = GetVar(env,"current_frame");
if (!cn.IsInt()) env->ThrowError("%s'current_frame' only available in runtime scripts",myName);
n = cn.AsInt(); // current_frame
}
n = (n<0) ? 0 : (n>=fc) ?fc-1:n; // Range limit frame n
}
if(ebf[0] == '\0') {
PVideoFrame src,dst;
src = child->GetFrame(n,env);
dst = env->NewVideoFrame(vi);
// use box size of x=4 by y=8
const boxh = (src->GetHeight(PLANAR_Y) + 8-1) / 8;
const boxw = (vi.width + 4-1) / 4; // vi.width, Planar or YUY2
int *counts= new int[boxh * boxw];
if (counts==NULL) strcpy(ebf,"counts memory alloc failure");
else if(!fpmd) {
ret = (yuy2) ? qCombed_YUY2(src,dst,n,Th,counts,boxh,boxw) : qCombed_Planar(src,dst,n,Th,counts,boxh,boxw);
} else {
int clipindex = args[3].AsInt(1); // ClipClop clip index 1 for combed frames
bool range= args[4].AsBool(false); // Single frames only
const char *rngsep_S = args[5].AsString("");
const int rngsep = (*rngsep_S<32 || *rngsep_S>126) ? ',' : *rngsep_S; // COMMA default
char FullPath[MAX_PATH];
if(_fullpath(FullPath,FileName, _MAX_PATH ) == NULL ) strcpy(ebf,"Cannot get full path.");
else {
if((fp=fopen(FullPath,"wt"))==NULL) strcpy(ebf,"Error opening dummy file for write");
else {
fclose(fp); // We will we be able to open file after scan
fp=NULL;
remove(FullPath); // kill dummy
char *cbf = new char[fc];
if(cbf==NULL) strcpy(ebf,"Cannot allocate comb buffer");
else {
memset(cbf,0,fc); // Prep for clip scan, clear combed flags
// ssS Modded to show timings
dprintf("Commencing combing scan");
double start_T = double(clock()) / double(CLOCKS_PER_SEC);
double sT = start_T;
int parts = min(20,fc);
int part=1;
int sf=0; // this time start frameno
int stopf = fc * part / parts;
double PercT=1.0; // anything
int cnt=0; // zero count of combed frames
for(n=0;n<fc;++n) {
src = child->GetFrame(n,env);
const bool q=(yuy2)
? qCombed_YUY2 (src,dst,n,Th,counts,boxh,boxw)
: qCombed_Planar(src,dst,n,Th,counts,boxh,boxw);
if(q) {
cbf[n]=1;
++cnt; // ++Combed count
}
if(n+1 >= stopf) {
double now = double(clock()) / double(CLOCKS_PER_SEC);
double Tim = max(now - sT,0.0001); // time taken for this part
double FPS = (n+1-sf)/Tim;
double SPF = 1.0/FPS;
if(sf==0) PercT=SPF; // 1st timing is 100.0%
double PartPerc=SPF * 100.0 / PercT; // Perc of 1st timing
dprintf("%d] %.2f%% nFrms=%d T=%.3fsec : %.2fFpS %.6fSpF Perc=%.2f%%",
n, (n+1)*100.0/fc ,n-sf+1,Tim,FPS,SPF,PartPerc);
sf=n+1; // next time start frame
sT=now; // next time start time
++part; // next time part number
stopf = fc * part / parts; // next time stop frame
}
}
ret = cnt; // Combed frames count for return
// Open text mode for Append
if((fp=fopen(FileName,"a+t"))==NULL) strcpy(ebf,"Cannot Open Frames file for write");
else {
double Tim = max(double(clock()) / double(CLOCKS_PER_SEC) - start_T,0.0001);
double FPS = fc/Tim;
double SPF = 1.0/FPS;
double AvePerc = SPF * 100.0 / PercT; // Perc of 1st timing
dprintf("Time=%.3fsecs (%.3fmins) : Avg %.3fFpS %.6fSpF Perc=%.2f%%",
Tim,Tim/60.0,FPS,SPF,AvePerc);
dprintf("Writing frames file");
char fbf[256];
if(!range) {
for(n=0;n<fc ;++n) {
if(cbf[n]!=0) {
if(clipindex<0) sprintf(fbf,"%d\n",n);
else sprintf(fbf,"%d %d\n",clipindex,n);
if(fwrite(fbf,strlen(fbf),1,fp)!=1) {
strcpy(ebf,"File write frame error");
break;
}
}
}
} else {
bool in=false;
int start = -1;
for(n=0; n<=fc ;++n) { // *** WARN ***, concludes at fc (FrameCount)
if(in && (n==fc || cbf[n]==0)) {
const int frms = n-start;
if(frms==1) {
if(clipindex<0) sprintf(fbf,"%d\n",start);
else sprintf(fbf,"%d %d\n",clipindex,start);
} else {
if(clipindex<0) sprintf(fbf,"%d%c%d\n",start,rngsep,n-1);
else sprintf(fbf,"%d %d%c%d\n",clipindex,start,rngsep,n-1);
}
if(fwrite(fbf,strlen(fbf),1,fp)!=1) {
strcpy(ebf,"File write range error");
break;
}
in = false;
start = -1;
} else if(!in && n<fc && cbf[n]!=0) {
start=n;
in=true;
}
}
}
}
delete [] cbf;
} // End of cbf==NULL
} // End of Open Dummy File
} // End of get FullPath
} // End of counts alloc
if(counts != NULL) {delete [] counts; counts=NULL;}
}
if(fp!=NULL) {fclose(fp); fp=NULL;}
if(ebf[0]!='\0') env->ThrowError("%s%s",myName,ebf);
return ret;
}
extern "C" __declspec(dllexport) const char* __stdcall AvisynthPluginInit2(IScriptEnvironment* env) {
env->AddFunction("qCombed" ,"c[Threshold]i[FileName]s[ClipIndex]i[Range]b[RngSep]s[n]i",qCombed , 0);
return "`qCombed' qCombed";
}
Question is, why is it slowing down over time ?
So far as I know, ColorBars keeps copy of same frame, so no cache problem there, and the called subroutine 'qCombed_Planar()' is doing exact same thing on each frame, and no memory allocs or anything being done within the concerned loop, the dst frame is from NewVideoFrame and re-used for entire scan (must provide to sub routine as reference otherwise problems if trying to take multiple write pointers on copy of writable video frame).
What is going on :confused:
Next post DebugView output
Here link to modded source and dll with timings code:- http://www.mediafire.com/download/vdpc647on11tabp/qCombed_v1.00_dll_Test.7z
Thanx in advance for your replies.
EDIT: Removed subroutine qCombed_YUY2 from source, too big for post (not of concern here).
EDIT: Small bug in above code (marked in red, fixed), frame counts off by 1 after 1st part(not a big problem).
sf=n+1; // next time start frame
sT=now; // next time start time
++part; // next time part number
stopf = fc * part / parts; // next time stop frame
EDIT: qCombed is a runtime function, not a filter (In File mode which we use here, will scan entire clip before return to script).
From docs
qCombed(), by StainlessS @ Doom9.org. http://forum.doom9.org/showthread.php?p=1776495#post1776495
Runtime function to establish combing status of video frames, or to write a frames file of combed frames [Intent for ClipClop() use].
ColorSpace, Planar and YUY2. (v2.5 dll but as only examines the luma channel can also function with v2.6 colorspaces).
qCombed() is a mod of IsCombed from DeComb plugin for Avisynth, (c) Donald Graft. [can be found here:- http://rationalqm.us/mine.html].
qCombed has two modes of operation, can use in conditional filter eg Scriptclip() or ConditionFilter() [just like IsCombed()],
and also has complete clip scan and frames file writing capability builtin.
Are you monitoring temperatures to check for thermal throttling?
Hi guys,
Getting some strange results where machine (Core Duo 2.4GHz, Duel Core, XP32SP3) slows over time doing exact same scans on YV12 ColorBars of duration 4 Hours.
Here Complete source to qCombed() v1.00 plugin:- (D9 thread http://forum.doom9.org/showthread.php?t=173754)
Only mods to published source are concerned with timing and DebugView output (mods marked in blue).
/*
qCombed() [ IsCombed() Mod by StainlessS @ Doom9.org.]
Mod of some of the source code in DeComb filter, FieldDeinterlace.cpp.
Original Decomb dll and source (c) Donald Graft, can be found here:- http://rationalqm.us/mine.html
*/
/*
qCombed() v1.0, 06 Aug 2016. First Release.
*/
#include <windows.h>
#include <stdio.h>
#include <time.h> // ssS Added for timings
#include "avisynth.h" // VERSION 3 header
// ssS Added
int __cdecl dprintf(char* fmt, ...) {
char printString[2048]="qCombed: ";
char *p=printString;
for(;*p++;);
--p; // @ null term
va_list argp;
va_start(argp, fmt);
vsprintf(p, fmt, argp);
va_end(argp);
for(;*p++;);
--p; // @ null term
if(printString == p || p[-1] != '\n') {
p[0]='\n'; // append n/l if not there already
p[1]='\0';
}
OutputDebugString(printString);
return p-printString; // strlen printString
}
AVSValue __cdecl GetVar(IScriptEnvironment* env, const char* name) {
try {return env->GetVar(name);} catch (IScriptEnvironment::NotFound) {} return AVSValue();
}
// Uses reference to dst else error (same dst for entire clip scan).
bool __cdecl qCombed_Planar(PVideoFrame src,PVideoFrame &dst,const int n,const int Th,int *counts,const int boxh,const int boxw) {
const BYTE *srcp = src->GetReadPtr(PLANAR_Y);
const int pitch = src->GetPitch(PLANAR_Y);
const w = src->GetRowSize(PLANAR_Y);
const h = src->GetHeight(PLANAR_Y);
// use box size of x=4 by y=8
const int boxarraysize = boxh * boxw;
const int CT = 4 * 8 / 2 - 1; // 15, (greater than this is at least half of max possible)
memset(counts, 0, sizeof(counts[0])*boxarraysize);
BYTE *dstp = dst->GetWritePtr(PLANAR_Y);
const int dpitch = dst->GetPitch(PLANAR_Y);
const dh = dst->GetHeight(PLANAR_Y);
const BYTE *above = srcp - pitch;
const BYTE *below = srcp + pitch;
memset(dstp,0,dh*dpitch); // dst is NewVideoFrame, ie not cropped.
int x, y;
for (y = 1; y < h - 1; ++y) {
srcp += pitch;
above += pitch;
below += pitch;
dstp += dpitch;
for (x = w; --x>=0;) {
const int val = ((long)above[x] - (long)srcp[x]) * ((long)below[x] - (long)srcp[x]);
if(val > Th) dstp[x] = 0xFF;
}
}
const BYTE *dp = dst->GetReadPtr(PLANAR_Y);
above = dp - dpitch;
below = dp + dpitch;
for (y = 1; y < h - 1; y++) {
const int boxy = (y >> 3) * boxw;
dp += dpitch;
above += dpitch;
below += dpitch;
for (x = w; --x>=0;) {
if (dp[x] == 0xFF && above[x] == 0xFF && below[x] == 0xFF) {
++counts[boxy + (x >> 2)];
}
}
}
bool ret=false;
for (x = boxarraysize; --x>=0;) {
if (counts[x] > CT) {ret = true; break;}
}
return ret;
}
AVSValue __cdecl qCombed(AVSValue args, void* user_data, IScriptEnvironment* env) {
char *myName="qCombed: ";
if(!args[0].IsClip()) env->ThrowError("%sMust have a clip",myName);
PClip child = args[0].AsClip();
const VideoInfo &vi=child->GetVideoInfo();
const bool yuy2 = vi.IsYUY2();
if(!vi.IsPlanar() && !yuy2) env->ThrowError("%sPlanar or YUY2 Only",myName);
if(vi.width==0||vi.num_frames<=0) env->ThrowError("%sClip has no video",myName);
int Th=args[1].AsInt(20);
if(Th<0 || Th>255) env->ThrowError("%sThreshold 0 -> 255",myName);
Th *= Th; // Squared
FILE *fp=NULL;
const fc=vi.num_frames;
int n=0;
const char *FileName = args[2].AsString("");
bool fpmd=(*FileName!='\0');
char ebf[256]=""; // No error so far
AVSValue ret;
if (!fpmd){ // Conditional filter function
if(args[6].IsInt()) {n = args[6].AsInt(); } // Frame n
else {
AVSValue cn = GetVar(env,"current_frame");
if (!cn.IsInt()) env->ThrowError("%s'current_frame' only available in runtime scripts",myName);
n = cn.AsInt(); // current_frame
}
n = (n<0) ? 0 : (n>=fc) ?fc-1:n; // Range limit frame n
}
if(ebf[0] == '\0') {
PVideoFrame src,dst;
src = child->GetFrame(n,env);
dst = env->NewVideoFrame(vi);
// use box size of x=4 by y=8
const boxh = (src->GetHeight(PLANAR_Y) + 8-1) / 8;
const boxw = (vi.width + 4-1) / 4; // vi.width, Planar or YUY2
int *counts= new int[boxh * boxw];
if (counts==NULL) strcpy(ebf,"counts memory alloc failure");
else if(!fpmd) {
ret = (yuy2) ? qCombed_YUY2(src,dst,n,Th,counts,boxh,boxw) : qCombed_Planar(src,dst,n,Th,counts,boxh,boxw);
} else {
int clipindex = args[3].AsInt(1); // ClipClop clip index 1 for combed frames
bool range= args[4].AsBool(false); // Single frames only
const char *rngsep_S = args[5].AsString("");
const int rngsep = (*rngsep_S<32 || *rngsep_S>126) ? ',' : *rngsep_S; // COMMA default
char FullPath[MAX_PATH];
if(_fullpath(FullPath,FileName, _MAX_PATH ) == NULL ) strcpy(ebf,"Cannot get full path.");
else {
if((fp=fopen(FullPath,"wt"))==NULL) strcpy(ebf,"Error opening dummy file for write");
else {
fclose(fp); // We will we be able to open file after scan
fp=NULL;
remove(FullPath); // kill dummy
char *cbf = new char[fc];
if(cbf==NULL) strcpy(ebf,"Cannot allocate comb buffer");
else {
memset(cbf,0,fc); // Prep for clip scan, clear combed flags
// ssS Modded to show timings
dprintf("Commencing combing scan");
double start_T = double(clock()) / double(CLOCKS_PER_SEC);
double sT = start_T;
int parts = min(20,fc);
int part=1;
int sf=0; // this time start frameno
int stopf = fc * part / parts;
double PercT=1.0; // anything
int cnt=0; // zero count of combed frames
for(n=0;n<fc;++n) {
src = child->GetFrame(n,env);
const bool q=(yuy2)
? qCombed_YUY2 (src,dst,n,Th,counts,boxh,boxw)
: qCombed_Planar(src,dst,n,Th,counts,boxh,boxw);
if(q) {
cbf[n]=1;
++cnt; // ++Combed count
}
if(n+1 >= stopf) {
double now = double(clock()) / double(CLOCKS_PER_SEC);
double Tim = max(now - sT,0.0001); // time taken for this part
double FPS = (n+1-sf)/Tim;
double SPF = 1.0/FPS;
if(sf==0) PercT=SPF; // 1st timing is 100.0%
double PartPerc=SPF * 100.0 / PercT; // Perc of 1st timing
dprintf("%d] %.2f%% nFrms=%d T=%.3fsec : %.2fFpS %.6fSpF Perc=%.2f%%",
n, (n+1)*100.0/fc ,n-sf+1,Tim,FPS,SPF,PartPerc);
sf=n+1; // next time start frame
sT=now; // next time start time
++part; // next time part number
stopf = fc * part / parts; // next time stop frame
}
}
ret = cnt; // Combed frames count for return
// Open text mode for Append
if((fp=fopen(FileName,"a+t"))==NULL) strcpy(ebf,"Cannot Open Frames file for write");
else {
double Tim = max(double(clock()) / double(CLOCKS_PER_SEC) - start_T,0.0001);
double FPS = fc/Tim;
double SPF = 1.0/FPS;
double AvePerc = SPF * 100.0 / PercT; // Perc of 1st timing
dprintf("Time=%.3fsecs (%.3fmins) : Avg %.3fFpS %.6fSpF Perc=%.2f%%",
Tim,Tim/60.0,FPS,SPF,AvePerc);
dprintf("Writing frames file");
char fbf[256];
if(!range) {
for(n=0;n<fc ;++n) {
if(cbf[n]!=0) {
if(clipindex<0) sprintf(fbf,"%d\n",n);
else sprintf(fbf,"%d %d\n",clipindex,n);
if(fwrite(fbf,strlen(fbf),1,fp)!=1) {
strcpy(ebf,"File write frame error");
break;
}
}
}
} else {
bool in=false;
int start = -1;
for(n=0; n<=fc ;++n) { // *** WARN ***, concludes at fc (FrameCount)
if(in && (n==fc || cbf[n]==0)) {
const int frms = n-start;
if(frms==1) {
if(clipindex<0) sprintf(fbf,"%d\n",start);
else sprintf(fbf,"%d %d\n",clipindex,start);
} else {
if(clipindex<0) sprintf(fbf,"%d%c%d\n",start,rngsep,n-1);
else sprintf(fbf,"%d %d%c%d\n",clipindex,start,rngsep,n-1);
}
if(fwrite(fbf,strlen(fbf),1,fp)!=1) {
strcpy(ebf,"File write range error");
break;
}
in = false;
start = -1;
} else if(!in && n<fc && cbf[n]!=0) {
start=n;
in=true;
}
}
}
}
delete [] cbf;
} // End of cbf==NULL
} // End of Open Dummy File
} // End of get FullPath
} // End of counts alloc
if(counts != NULL) {delete [] counts; counts=NULL;}
}
if(fp!=NULL) {fclose(fp); fp=NULL;}
if(ebf[0]!='\0') env->ThrowError("%s%s",myName,ebf);
return ret;
}
extern "C" __declspec(dllexport) const char* __stdcall AvisynthPluginInit2(IScriptEnvironment* env) {
env->AddFunction("qCombed" ,"c[Threshold]i[FileName]s[ClipIndex]i[Range]b[RngSep]s[n]i",qCombed , 0);
return "`qCombed' qCombed";
}
Question is, why is it slowing down over time ?
So far as I know, ColorBars keeps copy of same frame, so no cache problem there, and the called subroutine 'qCombed_Planar()' is doing exact same thing on each frame, and no memory allocs or anything being done within the concerned loop, the dst frame is from NewVideoFrame and re-used for entire scan (must provide to sub routine as reference otherwise problems if trying to take multiple write pointers on copy of writable video frame).
What is going on :confused:
Next post DebugView output
Here link to modded source and dll with timings code:- http://www.mediafire.com/download/vdpc647on11tabp/qCombed_v1.00_dll_Test.7z
Thanx in advance for your replies.
EDIT: Removed subroutine qCombed_YUY2 from source, too big for post (not of concern here).
EDIT: Small bug in above code (marked in red, fixed), frame counts off by 1 after 1st part(not a big problem).
sf=n+1; // next time start frame
sT=now; // next time start time
++part; // next time part number
stopf = fc * part / parts; // next time stop frame
EDIT: qCombed is a runtime function, not a filter (In File mode which we use here, will scan entire clip before return to script).
From docs
qCombed(), by StainlessS @ Doom9.org. http://forum.doom9.org/showthread.php?p=1776495#post1776495
Runtime function to establish combing status of video frames, or to write a frames file of combed frames [Intent for ClipClop() use].
ColorSpace, Planar and YUY2. (v2.5 dll but as only examines the luma channel can also function with v2.6 colorspaces).
qCombed() is a mod of IsCombed from DeComb plugin for Avisynth, (c) Donald Graft. [can be found here:- http://rationalqm.us/mine.html].
qCombed has two modes of operation, can use in conditional filter eg Scriptclip() or ConditionFilter() [just like IsCombed()],
and also has complete clip scan and frames file writing capability builtin.