about the problem in my transform function

[schweinsz]

I am coding a directshow-based h.264 decoder. I connect my decoder to the haali media splitter to play media files with h.264 contents. Most are fine. But some files including all avi-format files are not. When I render a avi file in graphedit, it connected to my decoder by the haali media splitter, but when I play it, the graphedit crashed.
I wrote some test code to locate the error and found the following function never run.
where does the error occur?

HRESULT CDiAVC::Transform(IMediaSample *pIn, IMediaSample *pOut)
{
bool retstatus = S_OK, isPure264 = 0;
CheckPointer(pIn,E_POINTER);
CheckPointer(pOut,E_POINTER);

if(handle == 0)
return S_FALSE;
if(pIn->IsDiscontinuity() == S_OK)
{
ll_flush_h264decoder(handle);
pOut->SetDiscontinuity(S_OK);
}
/*if (pIn->IsPreroll() == S_OK) //preroll samples should not be displayed
return S_OK;*/

ll_OutPutFrame *pDsplFrm = &outputFrm;
u_int8 *pBitstrm;
s_int32 totLength;
pIn->GetPointer(&pBitstrm);

bitStrm.pBitStream = pBitstrm;
totLength = pIn->GetActualDataLength();
if(((u_int32*)pBitstrm)[0] == 0x01000000)
isPure264 = 1;

while (totLength > 0)
{
u_int32 cur_len = (pBitstrm[0]<<24)+(pBitstrm[1]<<16)+(pBitstrm[2]<<8)+pBitstrm[3];
if(!isPure264) ((u_int32*)pBitstrm)[0] = 0x01000000;

bitStrm.bs_length = cur_len+4;
if(isPure264)
bitStrm.bs_length = totLength;
else if((u_int32)(bitStrm.bs_length) > totLength)
break;

writToFilew("dcss.264", (char*)(bitStrm.pBitStream), bitStrm.bs_length);

//the input buffer must include one slice at least
s_int32 status = ll_dec_h264decoder(handle, &bitStrm);

if(isPure264)
{
bitStrm.pBitStream += bitStrm.used_bytes;
totLength -= bitStrm.used_bytes;
}
else
{
bitStrm.pBitStream += (cur_len+4);
totLength -= (cur_len+4);
}
pBitstrm = bitStrm.pBitStream;

if(pDsplFrm->p_y_plane)
{
REFERENCE_TIME lasttime,dummy;
pIn->GetTime(&lasttime,&dummy);

if(!m_LastTime || (pIn->IsDiscontinuity() == S_OK) || lasttime < 0)
{
if(lasttime > 0 )
{
m_LastTime=lasttime;
}
else
{
m_LastTime+=m_FrameTime;
}
pOut->SetTime(NULL,NULL);
goto WriteYUVCurrPic;
}

REFERENCE_TIME endtime = m_LastTime + m_FrameTime;

pOut->SetTime(&m_LastTime,&endtime);
m_LastTime = endtime;

WriteYUVCurrPic:
u_int8 *dest_buffer;
if(pOut->GetPointer(&dest_buffer)!=S_OK)
return S_FALSE;

writeI420Tofile(dest_buffer, pDsplFrm);
pOut->SetActualDataLength(pDsplFrm->img_height * pDsplFrm->img_width * 3>>1);

retstatus = S_OK;
}
}
return retstatus;
}

[schweinsz]

some more code

u_int8* find_snalutype(u_int8 *start, u_int8 *ptr, u_int32 size, u_int32 type)
{
while(1)
{
if( ((ptr[0]&0x1f)==type) && ((ptr[0]>>7)==0) && ((ptr[0]&0x60)!=0) )
break;

ptr++;
if(ptr - start >= size)
return 0;
}
return ptr;
}
int sync_video_packet(const u_int8 *videobuf_code, u_int32 len)
{
if (len<4)
return 0;

while(1)
{
if(videobuf_code[0]==0 && videobuf_code[1]==0 && videobuf_code[2]==1)
break; // synced

videobuf_code++;
len--;
if (len==0)
return 0; //EOF
}
return 0x100|videobuf_code[3];
}
u_int32 nalu_encode(u_int8 *pdst, u_int8 *psrc, u_int32 len)
{
u_int8 *pdst1 = pdst, *pend = psrc + len;
u_int32 i_count = 0;

((u_int32*)pdst1)[0] = 0x01000000;
pdst1 += 4;

while( psrc < pend )
{
if( i_count == 2 && *psrc <= 0x03 )
{
*pdst1++ = 0x03;
i_count = 0;
}
if( *psrc == 0 )
i_count++;
else
i_count = 0;
*pdst1++ = *psrc++;
}
return pdst1 - pdst;
}
HRESULT CDiAVC::CheckInputType(const CMediaType *mtIn)
{
CheckPointer(mtIn,E_POINTER);
u_int8* p_spspps = 0;
s_int32 sizeLen;
HRESULT ret = E_FAIL;
DWORD fcc = 0;

dwPARx = dwPARy = 0;

if(handle == 0)
return VFW_E_TYPE_NOT_ACCEPTED;

// Can we transform this type
if(*mtIn->FormatType() == FORMAT_VideoInfo)
{
if(mtIn->cbFormat < sizeof(VIDEOINFOHEADER))
return E_INVALIDARG;
VIDEOINFOHEADER *pVih =
reinterpret_cast<VIDEOINFOHEADER*>(mtIn->pbFormat);

m_width = pVih->bmiHeader.biWidth;
m_height = abs(pVih->bmiHeader.biHeight);
m_FrameTime = pVih->AvgTimePerFrame;

sizeLen = mtIn->cbFormat-sizeof(VIDEOINFOHEADER);
p_spspps = sizeLen > 0 ?
(u_int8*)(mtIn->pbFormat + sizeof(VIDEOINFOHEADER)) : NULL;
fcc = pVih->bmiHeader.biCompression;
}
else if(*mtIn->FormatType() == FORMAT_VideoInfo2)
{
if(mtIn->cbFormat < sizeof(VIDEOINFOHEADER2))
return E_INVALIDARG;
VIDEOINFOHEADER2 *pVih =
reinterpret_cast<VIDEOINFOHEADER2*>(mtIn->pbFormat);

isInterlacedRawVideo = pVih->dwInterlaceFlags;
m_width = pVih->bmiHeader.biWidth;
m_height = abs(pVih->bmiHeader.biHeight);
m_FrameTime = pVih->AvgTimePerFrame;

sizeLen = mtIn->cbFormat-sizeof(VIDEOINFOHEADER2);
p_spspps = sizeLen > 0 ?
(u_int8*)(mtIn->pbFormat + sizeof(VIDEOINFOHEADER2)) : NULL;
fcc = pVih->bmiHeader.biCompression;
isVih2 = 1;
dwPARx = pVih->dwPictAspectRatioX;
dwPARy = pVih->dwPictAspectRatioY;
}
else if(*mtIn->FormatType() == FORMAT_MPEGVideo)
{
if(mtIn->cbFormat < sizeof(MPEG1VIDEOINFO))
return E_INVALIDARG;
MPEG1VIDEOINFO *pVih =
reinterpret_cast<MPEG1VIDEOINFO*>(mtIn->pbFormat);

m_width = pVih->hdr.bmiHeader.biWidth;
m_height = abs(pVih->hdr.bmiHeader.biHeight);
m_FrameTime = pVih->hdr.AvgTimePerFrame;

sizeLen = pVih->cbSequenceHeader;
p_spspps = sizeLen > 0 ?
(u_int8*)(pVih->bSequenceHeader) : NULL;
fcc = FOURCC_MPG1;
}
else if(*mtIn->FormatType() == FORMAT_MPEG2Video)
{
if(mtIn->cbFormat < sizeof(MPEG2VIDEOINFO))
return E_INVALIDARG;
MPEG2VIDEOINFO *pVih =
reinterpret_cast<MPEG2VIDEOINFO*>(mtIn->pbFormat);

isInterlacedRawVideo = pVih->hdr.dwInterlaceFlags;
m_width = pVih->hdr.bmiHeader.biWidth;
m_height = abs(pVih->hdr.bmiHeader.biHeight);
m_FrameTime = pVih->hdr.AvgTimePerFrame;

sizeLen = pVih->cbSequenceHeader;
p_spspps = sizeLen > 0 ?
(u_int8*)(pVih->dwSequenceHeader) : NULL;
if (pVih->hdr.bmiHeader.biCompression==0 || pVih->hdr.bmiHeader.biCompression == 0x0038002d)
fcc = FOURCC_MPG2;

isVih2 = 1;
dwPARx = pVih->hdr.dwPictAspectRatioX;
dwPARy = pVih->hdr.dwPictAspectRatioY;
}

if(m_FrameTime == 0)
m_FrameTime = 400000;
if (IsEqualGUID(*mtIn->Type(), MEDIATYPE_Video))
{
if( IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_AVC1) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_H264) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_h264) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_X264) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_x264) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_VSSH) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_avc1) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_H264VideoX1) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_H264_TRANSPORT) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_DAVC) ||
IsEqualGUID(*mtIn->Subtype(), MEDIASUBTYPE_PAVC))
{
ret = NOERROR;
}
else if(fcc==FOURCC_1001 || fcc==FOURCC_1002 || fcc==FOURCC_MPG1 || fcc==FOURCC_MPG2)
{
int sync=sync_video_packet(p_spspps, sizeLen);
if((sync&~0x60) == 0x107 && sync != 0x107) // H.264
ret = NOERROR;
}
}

if(p_spspps && ret==NOERROR)
{
u_int32 iii = 8;
u_int32 curlenz = sizeLen, isLenPre = 0;
u_int8 *pBitstrm = (u_int8*)malloc(sizeLen+16), *pbitss;
if(pBitstrm == 0)
return E_FAIL;

pbitss = find_snalutype(p_spspps, p_spspps, sizeLen, 7);
pbitss -= 2;
if(pbitss == 0 || pbitss<p_spspps)
goto EnDFreespsppsbuf;
curlenz = (pbitss[0]<<8)+pbitss[1];
if(curlenz == 1)
{
bitStrm.pBitStream = p_spspps;
bitStrm.bs_length = sizeLen;
isLenPre = 0;
}
else
{
bitStrm.pBitStream = pBitstrm;
bitStrm.bs_length = nalu_encode(pBitstrm, pbitss+2, curlenz);
isLenPre = 1;
}
s_int32 status = ll_dec_h264decoder(handle, &bitStrm);

if(isLenPre)
{
pbitss += (2+curlenz);
pbitss = find_snalutype(p_spspps, pbitss, sizeLen, 8);
pbitss -= 2;
if(pbitss == 0 || pbitss<p_spspps)
goto EnDFreespsppsbuf;
curlenz = (pbitss[0]<<8)+pbitss[1];

bitStrm.pBitStream = pBitstrm;
bitStrm.bs_length = nalu_encode(pBitstrm, pbitss+2, curlenz);
}
else
{
bitStrm.pBitStream += bitStrm.used_bytes;
bitStrm.bs_length -= bitStrm.used_bytes;
}

status = ll_dec_h264decoder(handle, &bitStrm);

if(outputFrm.img_height>0 && outputFrm.img_width>0)
{
m_width = outputFrm.img_width;
m_height = outputFrm.img_height;
}

EnDFreespsppsbuf:
free(pBitstrm);
}

return ret;
}