[pistacho]

I found the cause:

Some code related to AQ mode 4 is executed always.

This patch restores previous performance and not break anything (i think).

On file slicetype.cpp

line 481 replace

#define AQ_EDGE_BIAS 0.5
#define EDGE_INCLINATION 45
                uint32_t numCuInHeight = (maxRow + param->maxCUSize - 1) / param->maxCUSize;
                int maxHeight = numCuInHeight * param->maxCUSize;
                intptr_t stride = curFrame->m_fencPic->m_stride;
                pixel *edgePic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
                pixel *gaussianPic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
                pixel *thetaPic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
                memset(edgePic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
                memset(gaussianPic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
                memset(thetaPic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
                if (param->rc.aqMode == X265_AQ_EDGE)
                    edgeFilter(curFrame, edgePic, gaussianPic, thetaPic, stride, maxRow, maxCol);

                int blockXY = 0, inclinedEdge = 0;

with

#define AQ_EDGE_BIAS 0.5
#define EDGE_INCLINATION 45

		pixel *edgePic = NULL;
		pixel *gaussianPic = NULL;
		pixel *thetaPic = NULL;
				
		if (param->rc.aqMode == X265_AQ_EDGE)
		{
			uint32_t numCuInHeight = (maxRow + param->maxCUSize - 1) / param->maxCUSize;
			int maxHeight = numCuInHeight * param->maxCUSize;
			intptr_t stride = curFrame->m_fencPic->m_stride;
			edgePic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
			gaussianPic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
			thetaPic = X265_MALLOC(pixel, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)));
			memset(edgePic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
			memset(gaussianPic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
			memset(thetaPic, 0, stride * (maxHeight + (curFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));
			edgeFilter(curFrame, edgePic, gaussianPic, thetaPic, stride, maxRow, maxCol);
		}                  

                int blockXY = 0, inclinedEdge = 0;

and line 510

                     pixel *edgeImage = edgePic + curFrame->m_fencPic->m_lumaMarginY * stride + curFrame->m_fencPic->m_lumaMarginX;
                     pixel *edgeTheta = thetaPic + curFrame->m_fencPic->m_lumaMarginY * stride + curFrame->m_fencPic->m_lumaMarginX;

with

                     pixel *edgeImage = edgePic + curFrame->m_fencPic->m_lumaMarginY * curFrame->m_fencPic->m_stride + curFrame->m_fencPic->m_lumaMarginX;
                     pixel *edgeTheta = thetaPic + curFrame->m_fencPic->m_lumaMarginY * curFrame->m_fencPic->m_stride + curFrame->m_fencPic->m_lumaMarginX;

and line 545

                X265_FREE(edgePic);
                X265_FREE(gaussianPic);
                X265_FREE(thetaPic);

with

		if (param->rc.aqMode == X265_AQ_EDGE)
		{
			X265_FREE(edgePic);
			X265_FREE(gaussianPic);
			X265_FREE(thetaPic);
		}

And binary with this patch applied: x265 v3.2 patched x64 GCC 9.2.0