lgladsden/jellyfin-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3442 from nyanmisaka/tonemap

Browse Source 
Tonemapping function relying on OpenCL filter and NVENC HEVC decoder
This commit is contained in:
Joshua M. Boniface 2020-09-04 10:06:15 -04:00 committed by GitHub
commit 559a7fc336
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 187 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
View File

@ -456,6 +456,7 @@ namespace MediaBrowser.Controller.MediaEncoding
var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1;
var isQsvDecoder = videoDecoder.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1;
var isQsvEncoder = outputVideoCodec.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1;
var isNvencHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
var isMacOS = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
@ -515,6 +516,24 @@ namespace MediaBrowser.Controller.MediaEncoding
}
}
if (state.IsVideoRequest
&& string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase))
{
var isColorDepth10 = IsColorDepth10(state);
if (isNvencHevcDecoder && isColorDepth10
&& _mediaEncoder.SupportsHwaccel("opencl")
&& encodingOptions.EnableTonemapping
&& !string.IsNullOrEmpty(state.VideoStream.VideoRange)
&& state.VideoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
{
arg.Append("-init_hw_device opencl=ocl:")
.Append(encodingOptions.OpenclDevice)
.Append(' ')
.Append("-filter_hw_device ocl ");
}
}
if (state.IsVideoRequest
&& string.Equals(encodingOptions.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase))
{
@ -1003,11 +1022,33 @@ namespace MediaBrowser.Controller.MediaEncoding
if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase))
{
param = "-pix_fmt yuv420p " + param;
}
if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase))
{
var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty;
var videoStream = state.VideoStream;
var isColorDepth10 = IsColorDepth10(state);
if (videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1
&& isColorDepth10
&& _mediaEncoder.SupportsHwaccel("opencl")
&& encodingOptions.EnableTonemapping
&& !string.IsNullOrEmpty(videoStream.VideoRange)
&& videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
{
param = "-pix_fmt nv12 " + param;
}
else
{
param = "-pix_fmt yuv420p " + param;
}
}
if (string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase))
{
param = "-pix_fmt nv21 " + param;
@ -1611,64 +1652,45 @@ namespace MediaBrowser.Controller.MediaEncoding
var outputSizeParam = ReadOnlySpan<char>.Empty;
var request = state.BaseRequest;
// Add resolution params, if specified
if (request.Width.HasValue
|| request.Height.HasValue
|| request.MaxHeight.HasValue
|| request.MaxWidth.HasValue)
outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"');
// All possible beginning of video filters
// Don't break the order
string[] beginOfOutputSizeParam = new[]
{
outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"');
// for tonemap_opencl
"hwupload,tonemap_opencl",
// hwupload=extra_hw_frames=64,vpp_qsv (for overlay_qsv on linux)
var index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase);
"hwupload=extra_hw_frames",
// vpp_qsv
"vpp",
// hwdownload,format=p010le (hardware decode + software encode for vaapi)
"hwdownload",
// format=nv12|vaapi,hwupload,scale_vaapi
"format",
// bwdif,scale=expr
"bwdif",
// yadif,scale=expr
"yadif",
// scale=expr
"scale"
};
var index = -1;
foreach (var param in beginOfOutputSizeParam)
{
index = outputSizeParam.IndexOf(param, StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
else
{
// vpp_qsv
index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
else
{
// hwdownload,format=p010le (hardware decode + software encode for vaapi)
index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
else
{
// format=nv12|vaapi,hwupload,scale_vaapi
index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
else
{
// yadif,scale=expr
index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
else
{
// scale=expr
index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase);
if (index != -1)
{
outputSizeParam = outputSizeParam.Slice(index);
}
}
}
}
}
break;
}
}
@ -1747,9 +1769,9 @@ namespace MediaBrowser.Controller.MediaEncoding
*/
if (isLinux)
{
retStr = !outputSizeParam.IsEmpty ?
" -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\"" :
" -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\"";
retStr = !outputSizeParam.IsEmpty
? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\""
: " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\"";
}
}
@ -2084,8 +2106,10 @@ namespace MediaBrowser.Controller.MediaEncoding
var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1;
var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1;
var isNvdecH264Decoder = videoDecoder.IndexOf("h264_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
var isNvdecHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
var isLibX264Encoder = outputVideoCodec.IndexOf("libx264", StringComparison.OrdinalIgnoreCase) != -1;
var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
var isColorDepth10 = IsColorDepth10(state);
var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
@ -2093,6 +2117,50 @@ namespace MediaBrowser.Controller.MediaEncoding
// If double rate deinterlacing is enabled and the input framerate is 30fps or below, otherwise the output framerate will be too high for many devices
var doubleRateDeinterlace = options.DeinterlaceDoubleRate && (videoStream?.RealFrameRate ?? 60) <= 30;
// Currently only with the use of NVENC decoder can we get a decent performance.
// Currently only the HEVC/H265 format is supported.
// NVIDIA Pascal and Turing or higher are recommended.
if (isNvdecHevcDecoder && isColorDepth10
&& _mediaEncoder.SupportsHwaccel("opencl")
&& options.EnableTonemapping
&& !string.IsNullOrEmpty(videoStream.VideoRange)
&& videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
{
var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}";
if (options.TonemappingParam != 0)
{
parameters += ":param={4}";
}
if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase))
{
parameters += ":range={5}";
}
// Upload the HDR10 or HLG data to the OpenCL device,
// use tonemap_opencl filter for tone mapping,
// and then download the SDR data to memory.
filters.Add("hwupload");
filters.Add(
string.Format(
CultureInfo.InvariantCulture,
parameters,
options.TonemappingAlgorithm,
options.TonemappingDesat,
options.TonemappingThreshold,
options.TonemappingPeak,
options.TonemappingParam,
options.TonemappingRange));
filters.Add("hwdownload");
if (hasGraphicalSubs || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true)
|| string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
{
filters.Add("format=nv12");
}
}
// When the input may or may not be hardware VAAPI decodable
if (isVaapiH264Encoder)
{
@ -2110,7 +2178,6 @@ namespace MediaBrowser.Controller.MediaEncoding
else if (IsVaapiSupported(state) && isVaapiDecoder && isLibX264Encoder)
{
var codec = videoStream.Codec.ToLowerInvariant();
var isColorDepth10 = IsColorDepth10(state);
// Assert 10-bit hardware VAAPI decodable
if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)

14
MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs
View File

@ -282,6 +282,20 @@ namespace MediaBrowser.MediaEncoding.Probing
[JsonPropertyName("disposition")]
public IReadOnlyDictionary<string, int> Disposition { get; set; }
/// <summary>
/// Gets or sets the color range.
/// </summary>
/// <value>The color range.</value>
[JsonPropertyName("color_range")]
public string ColorRange { get; set; }
/// <summary>
/// Gets or sets the color space.
/// </summary>
/// <value>The color space.</value>
[JsonPropertyName("color_space")]
public string ColorSpace { get; set; }
/// <summary>
/// Gets or sets the color transfer.
/// </summary>

10
MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
View File

@ -714,6 +714,16 @@ namespace MediaBrowser.MediaEncoding.Probing
stream.RefFrames = streamInfo.Refs;
}
if (!string.IsNullOrEmpty(streamInfo.ColorRange))
{
stream.ColorRange = streamInfo.ColorRange;
}
if (!string.IsNullOrEmpty(streamInfo.ColorSpace))
{
stream.ColorSpace = streamInfo.ColorSpace;
}
if (!string.IsNullOrEmpty(streamInfo.ColorTransfer))
{
stream.ColorTransfer = streamInfo.ColorTransfer;

29
MediaBrowser.Model/Configuration/EncodingOptions.cs
View File

@ -31,6 +31,22 @@ namespace MediaBrowser.Model.Configuration
public string VaapiDevice { get; set; }
public string OpenclDevice { get; set; }
public bool EnableTonemapping { get; set; }
public string TonemappingAlgorithm { get; set; }
public string TonemappingRange { get; set; }
public double TonemappingDesat { get; set; }
public double TonemappingThreshold { get; set; }
public double TonemappingPeak { get; set; }
public double TonemappingParam { get; set; }
public int H264Crf { get; set; }
public int H265Crf { get; set; }
@ -58,8 +74,19 @@ namespace MediaBrowser.Model.Configuration
EnableThrottling = false;
ThrottleDelaySeconds = 180;
EncodingThreadCount = -1;
// This is a DRM device that is almost guaranteed to be there on every intel platform, plus it's the default one in ffmpeg if you don't specify anything
// This is a DRM device that is almost guaranteed to be there on every intel platform,
// plus it's the default one in ffmpeg if you don't specify anything
VaapiDevice = "/dev/dri/renderD128";
// This is the OpenCL device that is used for tonemapping.
// The left side of the dot is the platform number, and the right side is the device number on the platform.
OpenclDevice = "0.0";
EnableTonemapping = false;
TonemappingAlgorithm = "reinhard";
TonemappingRange = "auto";
TonemappingDesat = 0;
TonemappingThreshold = 0.8;
TonemappingPeak = 0;
TonemappingParam = 0;
H264Crf = 23;
H265Crf = 28;
DeinterlaceDoubleRate = false;

18
MediaBrowser.Model/Entities/MediaStream.cs
View File

@ -35,6 +35,18 @@ namespace MediaBrowser.Model.Entities
/// <value>The language.</value>
public string Language { get; set; }
/// <summary>
/// Gets or sets the color range.
/// </summary>
/// <value>The color range.</value>
public string ColorRange { get; set; }
/// <summary>
/// Gets or sets the color space.
/// </summary>
/// <value>The color space.</value>
public string ColorSpace { get; set; }
/// <summary>
/// Gets or sets the color transfer.
/// </summary>
@ -47,12 +59,6 @@ namespace MediaBrowser.Model.Entities
/// <value>The color primaries.</value>
public string ColorPrimaries { get; set; }
/// <summary>
/// Gets or sets the color space.
/// </summary>
/// <value>The color space.</value>
public string ColorSpace { get; set; }
/// <summary>
/// Gets or sets the comment.
/// </summary>