I optimized these shaders and have given them multi-stage controls over the noise thresholds. Note that I can still expand the search area beyond the current 9 horizontal and vertical pixels if you wish (though I doubt the effect would gain much from that).
Use the renderer option "Disable Initial Color Mixing Stages" and set up these shaders as follows:
1- R'G'B' to Y'CbCr for LMS rendering
2- if required (on Intel/AMD graphics adapters), an up-sampling shader for 4:2:0 to 4:2:2
3- if required (on Intel/AMD graphics adapters), an up-sampling shader for 4:2:2 to 4:4:4
4- horizontal pass sharpen complex, deband and denoise for LMS rendering
5- vertical pass sharpen complex, deband and denoise, and color conversion to LMS for LMS rendering// (C) 2015 Jan-Willem Krans (janwillem32 <at> hotmail.com)
// This file is part of Video pixel shader pack.
// This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
// R'G'B' to Y'CbCr for LMS rendering
// This shader should be run as the very first pixel shader in the chain.
// This shader requires compiling with ps_2_0, but higher is better, see http://en.wikipedia.org/wiki/Pixel_shader to look up what PS version your video card supports.
#ifndef Ml// external compiler
#define Ml 0
#define Mi 0
#define Mr 0
#endif
#if Ml
#define tex2D(s, t) tex2Dlod(s, float4(t, 0., 0.))
#endif
sampler s0 : register(s0);
static const float3x3 mty = float3x3(
#if Mi == 0
.2126, -1063./9278., .5, .7152, -1788./4639., -1788./3937., .0722, .5, -361./7874.// BT.709 (+BT.2020) R'G'B' to Y'CbCr
#elif Mi == 1
.299, -299./1772., .5, .587, -587./1772., -587./1402., .114, .5, -57./701.// BT.601 (+BT.2020) R'G'B' to Y'CbCr
#else
.212, -106./913., .5, .701, -701./1576., -701./1576., .087, .5, -87./1576.// SMPTE 240M R'G'B' to Y'CbCr
#endif
)
#if Mr
*32767./65535.// convert to alternative limited ranges, first part
#endif
;
float4 main(float2 tex : TEXCOORD0) : COLOR
{
float3 s1 = tex2D(s0, tex).rgb;// original pixel
s1 = s1.r*mty[0]+s1.g*mty[1]+s1.b*mty[2];// convert to Y'CbCr
#if Mr
s1 += float2(16384./65535., 32767./65535.).xyy;// convert to alternative limited ranges, second part
#endif
return s1.rgbb;
}// (C) 2015 Jan-Willem Krans (janwillem32 <at> hotmail.com)
// This file is part of Video pixel shader pack.
// This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
// horizontal pass sharpen complex, deband and denoise for LMS rendering
// This shader should be run as a Y'CbCr-stage pixel shader.
// This shader requires compiling with ps_3_0, see http://en.wikipedia.org/wiki/Pixel_shader to look up what PS version your video card supports.
// fractions, either decimal or not, are allowed
#define SharpenLimitLuma 2// valid interval [0, 10], luma-specific sharpening limit, 0 is disabled, lower numbers will allow more sharpening on contours
#define SharpenLimitChroma 2// valid interval [0, 10], chroma-specific sharpening limit, 0 is disabled, lower numbers will allow more sharpening on contours
#define LumaDetectionFactor 64// valid interval (1, 250], luma-specific detection factor, if set to the lowest amount no contours can be detected, higher numbers will shift the detection on color difference intervals of debanding to noise detection limit to mimimum sharpening to maximum sharpening toward more sharpening
#define ChromaDetectionFactor 64// valid interval (1, 250], chroma-specific detection factor, if set to the lowest amount no contours can be detected, higher numbers will shift the detection on color difference intervals of debanding to noise detection limit to mimimum sharpening to maximum sharpening toward more sharpening
// valid intervals [0, 1), banding treshold, higher numbers mean stronger deband and denoise
#define NoiseThreshold1 .015625// innermost threshold, mostly affects denoising
#define NoiseThreshold2 .015625
#define NoiseThreshold3 .015625
#define NoiseThreshold4 .015625// outermost threshold, mostly affects debanding area
#ifndef Ml// external compiler
#define Mr 0
#endif
sampler s0 : register(s0);
float2 c1 : register(c1);
#define sp(a) tex2Dlod(s0, float4(tex+c1*float2(a, 0), 0, 0)).rgb
static const float3 slimits = float3(-SharpenLimitLuma, -SharpenLimitChroma, -SharpenLimitChroma);
static const float3 dfactors = float3(LumaDetectionFactor, ChromaDetectionFactor, ChromaDetectionFactor)*(32768./32767.*float(Mr)+1.);
float4 main(float2 tex : TEXCOORD0) : COLOR
{
float3 n, p, s1 = sp(0);// original pixel
{
float3 s2 = sp(-1);
float3 af = 1.;// accumulated amount of colors from the samples
float3 ac = s1;// accumulate color
float3 cd = abs(s1-s2);// color difference
float3 rcd = max(slimits, 1.-dfactors*cd);// factor for both base and multiplicand is 1.0, the output will be in the interval (-inf, 1]
// invert interval on sharpening
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s2*rcd;
[branch] if(!any(cd >= NoiseThreshold1*(1.-32768./65535.*float(Mr)))) {// continue if all channels are below the noise threshold
float3 s3 = sp(-2);
cd = abs(s1-s3);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s3*rcd;
[branch] if(!any(cd >= NoiseThreshold2*(1.-32768./65535.*float(Mr)))) {
float3 s4 = sp(-3);
cd = abs(s1-s4);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s4*rcd;
[branch] if(!any(cd >= NoiseThreshold3*(1.-32768./65535.*float(Mr)))) {
float3 s5 = sp(-4);
cd = abs(s1-s5);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s5*rcd;
[branch] if(!any(cd >= NoiseThreshold4*(1.-32768./65535.*float(Mr)))) {
float3 s6 = sp(-5);
cd = abs(s1-s6);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s6*rcd;
}
}
}
}
n = ac/af;
}
{
float3 s2 = sp(1);
float3 af = 1.;// accumulated amount of colors from the samples
float3 ac = s1;// accumulate color
float3 cd = abs(s1-s2);// color difference
float3 rcd = max(slimits, 1.-dfactors*cd);// factor for both base and multiplicand is 1.0, the output will be in the interval (-inf, 1]
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s2*rcd;
[branch] if(!any(cd >= NoiseThreshold1*(1.-32768./65535.*float(Mr)))) {// continue if all channels are below the noise threshold
float3 s3 = sp(2);
cd = abs(s1-s3);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s3*rcd;
[branch] if(!any(cd >= NoiseThreshold2*(1.-32768./65535.*float(Mr)))) {
float3 s4 = sp(3);
cd = abs(s1-s4);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s4*rcd;
[branch] if(!any(cd >= NoiseThreshold3*(1.-32768./65535.*float(Mr)))) {
float3 s5 = sp(4);
cd = abs(s1-s5);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s5*rcd;
[branch] if(!any(cd >= NoiseThreshold4*(1.-32768./65535.*float(Mr)))) {
float3 s6 = sp(5);
cd = abs(s1-s6);
rcd = max(slimits, 1.-dfactors*cd);
if(rcd.x < 0) rcd.x = SharpenLimitLuma-abs(rcd.x);
if(rcd.y < 0) rcd.y = SharpenLimitChroma-abs(rcd.y);
if(rcd.z < 0) rcd.z = SharpenLimitChroma-abs(rcd.z);
af += abs(rcd);
ac += s6*rcd;
}
}
}
}
p = ac/af;
}
return ((n+p)*.5).rgbb;
}