using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Api.Extensions; using Jellyfin.Data.Enums; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Streaming; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; namespace Jellyfin.Api.Helpers; /// /// The streaming helpers. /// public static class StreamingHelpers { /// /// Gets the current streaming state. /// /// The . /// The . /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of . /// Instance of the interface. /// The . /// The . /// A containing the current . public static async Task GetStreamingState( StreamingRequestDto streamingRequest, HttpContext httpContext, IMediaSourceManager mediaSourceManager, IUserManager userManager, ILibraryManager libraryManager, IServerConfigurationManager serverConfigurationManager, IMediaEncoder mediaEncoder, EncodingHelper encodingHelper, ITranscodeManager transcodeManager, TranscodingJobType transcodingJobType, CancellationToken cancellationToken) { var httpRequest = httpContext.Request; if (!string.IsNullOrWhiteSpace(streamingRequest.Params)) { ParseParams(streamingRequest); } streamingRequest.StreamOptions = ParseStreamOptions(httpRequest.Query); if (httpRequest.Path.Value is null) { throw new ResourceNotFoundException(nameof(httpRequest.Path)); } var url = httpRequest.Path.Value.AsSpan().RightPart('.').ToString(); if (string.IsNullOrEmpty(streamingRequest.AudioCodec)) { streamingRequest.AudioCodec = encodingHelper.InferAudioCodec(url); } var state = new StreamState(mediaSourceManager, transcodingJobType, transcodeManager) { Request = streamingRequest, RequestedUrl = url, UserAgent = httpRequest.Headers[HeaderNames.UserAgent] }; var userId = httpContext.User.GetUserId(); if (!userId.IsEmpty()) { state.User = userManager.GetUserById(userId); } if (state.IsVideoRequest && !string.IsNullOrWhiteSpace(state.Request.VideoCodec)) { state.SupportedVideoCodecs = state.Request.VideoCodec.Split(',', StringSplitOptions.RemoveEmptyEntries); state.Request.VideoCodec = state.SupportedVideoCodecs.FirstOrDefault(); } if (!string.IsNullOrWhiteSpace(streamingRequest.AudioCodec)) { state.SupportedAudioCodecs = streamingRequest.AudioCodec.Split(',', StringSplitOptions.RemoveEmptyEntries); state.Request.AudioCodec = state.SupportedAudioCodecs.FirstOrDefault(mediaEncoder.CanEncodeToAudioCodec) ?? state.SupportedAudioCodecs.FirstOrDefault(); } if (!string.IsNullOrWhiteSpace(streamingRequest.SubtitleCodec)) { state.SupportedSubtitleCodecs = streamingRequest.SubtitleCodec.Split(',', StringSplitOptions.RemoveEmptyEntries); state.Request.SubtitleCodec = state.SupportedSubtitleCodecs.FirstOrDefault(mediaEncoder.CanEncodeToSubtitleCodec) ?? state.SupportedSubtitleCodecs.FirstOrDefault(); } var item = libraryManager.GetItemById(streamingRequest.Id) ?? throw new ResourceNotFoundException(); state.IsInputVideo = item.MediaType == MediaType.Video; MediaSourceInfo? mediaSource = null; if (string.IsNullOrWhiteSpace(streamingRequest.LiveStreamId)) { var currentJob = !string.IsNullOrWhiteSpace(streamingRequest.PlaySessionId) ? transcodeManager.GetTranscodingJob(streamingRequest.PlaySessionId) : null; if (currentJob is not null) { mediaSource = currentJob.MediaSource; } if (mediaSource is null) { var mediaSources = await mediaSourceManager.GetPlaybackMediaSources(libraryManager.GetItemById(streamingRequest.Id), null, false, false, cancellationToken).ConfigureAwait(false); mediaSource = string.IsNullOrEmpty(streamingRequest.MediaSourceId) ? mediaSources[0] : mediaSources.Find(i => string.Equals(i.Id, streamingRequest.MediaSourceId, StringComparison.Ordinal)); if (mediaSource is null && Guid.Parse(streamingRequest.MediaSourceId).Equals(streamingRequest.Id)) { mediaSource = mediaSources[0]; } } } else { // Enforce more restrictive transcoding profile for LiveTV due to compatability reasons // Cap the MaxStreamingBitrate to 30Mbps, because we are unable to reliably probe source bitrate, // which will cause the client to request extremely high bitrate that may fail the player/encoder streamingRequest.VideoBitRate = streamingRequest.VideoBitRate > 30000000 ? 30000000 : streamingRequest.VideoBitRate; if (streamingRequest.SegmentContainer is not null) { // Remove all fmp4 transcoding profiles, because it causes playback error and/or A/V sync issues // Notably: Some channels won't play on FireFox and LG webOS // Some channels from HDHomerun will experience A/V sync issues streamingRequest.SegmentContainer = "ts"; streamingRequest.VideoCodec = "h264"; streamingRequest.AudioCodec = "aac"; state.SupportedVideoCodecs = ["h264"]; state.Request.VideoCodec = "h264"; state.SupportedAudioCodecs = ["aac"]; state.Request.AudioCodec = "aac"; } var liveStreamInfo = await mediaSourceManager.GetLiveStreamWithDirectStreamProvider(streamingRequest.LiveStreamId, cancellationToken).ConfigureAwait(false); mediaSource = liveStreamInfo.Item1; state.DirectStreamProvider = liveStreamInfo.Item2; } var encodingOptions = serverConfigurationManager.GetEncodingOptions(); encodingHelper.AttachMediaSourceInfo(state, encodingOptions, mediaSource, url); string? containerInternal = Path.GetExtension(state.RequestedUrl); if (!string.IsNullOrEmpty(streamingRequest.Container)) { containerInternal = streamingRequest.Container; } if (string.IsNullOrEmpty(containerInternal)) { containerInternal = streamingRequest.Static ? StreamBuilder.NormalizeMediaSourceFormatIntoSingleContainer(state.InputContainer, null, DlnaProfileType.Audio) : GetOutputFileExtension(state, mediaSource); } var outputAudioCodec = streamingRequest.AudioCodec; state.OutputAudioCodec = outputAudioCodec; state.OutputContainer = (containerInternal ?? string.Empty).TrimStart('.'); state.OutputAudioChannels = encodingHelper.GetNumAudioChannelsParam(state, state.AudioStream, state.OutputAudioCodec); if (EncodingHelper.LosslessAudioCodecs.Contains(outputAudioCodec)) { state.OutputAudioBitrate = state.AudioStream.BitRate ?? 0; } else { state.OutputAudioBitrate = encodingHelper.GetAudioBitrateParam(streamingRequest.AudioBitRate, streamingRequest.AudioCodec, state.AudioStream, state.OutputAudioChannels) ?? 0; } if (outputAudioCodec.StartsWith("pcm_", StringComparison.Ordinal)) { containerInternal = ".pcm"; } if (state.VideoRequest is not null) { state.OutputVideoCodec = state.Request.VideoCodec; state.OutputVideoBitrate = encodingHelper.GetVideoBitrateParamValue(state.VideoRequest, state.VideoStream, state.OutputVideoCodec); encodingHelper.TryStreamCopy(state); if (!EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && state.OutputVideoBitrate.HasValue) { var isVideoResolutionNotRequested = !state.VideoRequest.Width.HasValue && !state.VideoRequest.Height.HasValue && !state.VideoRequest.MaxWidth.HasValue && !state.VideoRequest.MaxHeight.HasValue; if (isVideoResolutionNotRequested && state.VideoStream is not null && state.VideoRequest.VideoBitRate.HasValue && state.VideoStream.BitRate.HasValue && state.VideoRequest.VideoBitRate.Value >= state.VideoStream.BitRate.Value) { // Don't downscale the resolution if the width/height/MaxWidth/MaxHeight is not requested, // and the requested video bitrate is higher than source video bitrate. if (state.VideoStream.Width.HasValue || state.VideoStream.Height.HasValue) { state.VideoRequest.MaxWidth = state.VideoStream?.Width; state.VideoRequest.MaxHeight = state.VideoStream?.Height; } } else { var resolution = ResolutionNormalizer.Normalize( state.VideoStream?.BitRate, state.OutputVideoBitrate.Value, state.VideoRequest.MaxWidth, state.VideoRequest.MaxHeight); state.VideoRequest.MaxWidth = resolution.MaxWidth; state.VideoRequest.MaxHeight = resolution.MaxHeight; } } } var ext = string.IsNullOrWhiteSpace(state.OutputContainer) ? GetOutputFileExtension(state, mediaSource) : ("." + GetContainerFileExtension(state.OutputContainer)); state.OutputFilePath = GetOutputFilePath(state, ext, serverConfigurationManager, streamingRequest.DeviceId, streamingRequest.PlaySessionId); return state; } /// /// Parses query parameters as StreamOptions. /// /// The query string. /// A containing the stream options. private static Dictionary ParseStreamOptions(IQueryCollection queryString) { Dictionary streamOptions = new Dictionary(); foreach (var param in queryString) { if (char.IsLower(param.Key[0])) { // This was probably not parsed initially and should be a StreamOptions // or the generated URL should correctly serialize it // TODO: This should be incorporated either in the lower framework for parsing requests streamOptions[param.Key] = param.Value; } } return streamOptions; } /// /// Gets the output file extension. /// /// The state. /// The mediaSource. /// System.String. private static string GetOutputFileExtension(StreamState state, MediaSourceInfo? mediaSource) { var ext = Path.GetExtension(state.RequestedUrl); if (!string.IsNullOrEmpty(ext)) { return ext; } // Try to infer based on the desired video codec if (state.IsVideoRequest) { var videoCodec = state.Request.VideoCodec; if (string.Equals(videoCodec, "h264", StringComparison.OrdinalIgnoreCase)) { return ".ts"; } if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase) || string.Equals(videoCodec, "av1", StringComparison.OrdinalIgnoreCase)) { return ".mp4"; } if (string.Equals(videoCodec, "theora", StringComparison.OrdinalIgnoreCase)) { return ".ogv"; } if (string.Equals(videoCodec, "vp8", StringComparison.OrdinalIgnoreCase) || string.Equals(videoCodec, "vp9", StringComparison.OrdinalIgnoreCase) || string.Equals(videoCodec, "vpx", StringComparison.OrdinalIgnoreCase)) { return ".webm"; } if (string.Equals(videoCodec, "wmv", StringComparison.OrdinalIgnoreCase)) { return ".asf"; } } else { // Try to infer based on the desired audio codec var audioCodec = state.Request.AudioCodec; if (string.Equals("aac", audioCodec, StringComparison.OrdinalIgnoreCase)) { return ".aac"; } if (string.Equals("mp3", audioCodec, StringComparison.OrdinalIgnoreCase)) { return ".mp3"; } if (string.Equals("vorbis", audioCodec, StringComparison.OrdinalIgnoreCase)) { return ".ogg"; } if (string.Equals("wma", audioCodec, StringComparison.OrdinalIgnoreCase)) { return ".wma"; } } // Fallback to the container of mediaSource if (!string.IsNullOrEmpty(mediaSource?.Container)) { var idx = mediaSource.Container.IndexOf(',', StringComparison.OrdinalIgnoreCase); return '.' + (idx == -1 ? mediaSource.Container : mediaSource.Container[..idx]).Trim(); } throw new InvalidOperationException("Failed to find an appropriate file extension"); } /// /// Gets the output file path for transcoding. /// /// The current . /// The file extension of the output file. /// Instance of the interface. /// The device id. /// The play session id. /// The complete file path, including the folder, for the transcoding file. private static string GetOutputFilePath(StreamState state, string outputFileExtension, IServerConfigurationManager serverConfigurationManager, string? deviceId, string? playSessionId) { var data = $"{state.MediaPath}-{state.UserAgent}-{deviceId!}-{playSessionId!}"; var filename = data.GetMD5().ToString("N", CultureInfo.InvariantCulture); var ext = outputFileExtension.ToLowerInvariant(); var folder = serverConfigurationManager.GetTranscodePath(); return Path.Combine(folder, filename + ext); } /// /// Parses the parameters. /// /// The request. private static void ParseParams(StreamingRequestDto request) { if (string.IsNullOrEmpty(request.Params)) { return; } var vals = request.Params.Split(';'); var videoRequest = request as VideoRequestDto; for (var i = 0; i < vals.Length; i++) { var val = vals[i]; if (string.IsNullOrWhiteSpace(val)) { continue; } switch (i) { case 0: // DeviceProfileId break; case 1: request.DeviceId = val; break; case 2: request.MediaSourceId = val; break; case 3: request.Static = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); break; case 4: if (videoRequest is not null) { videoRequest.VideoCodec = val; } break; case 5: request.AudioCodec = val; break; case 6: if (videoRequest is not null) { videoRequest.AudioStreamIndex = int.Parse(val, CultureInfo.InvariantCulture); } break; case 7: if (videoRequest is not null) { videoRequest.SubtitleStreamIndex = int.Parse(val, CultureInfo.InvariantCulture); } break; case 8: if (videoRequest is not null) { videoRequest.VideoBitRate = int.Parse(val, CultureInfo.InvariantCulture); } break; case 9: request.AudioBitRate = int.Parse(val, CultureInfo.InvariantCulture); break; case 10: request.MaxAudioChannels = int.Parse(val, CultureInfo.InvariantCulture); break; case 11: if (videoRequest is not null) { videoRequest.MaxFramerate = float.Parse(val, CultureInfo.InvariantCulture); } break; case 12: if (videoRequest is not null) { videoRequest.MaxWidth = int.Parse(val, CultureInfo.InvariantCulture); } break; case 13: if (videoRequest is not null) { videoRequest.MaxHeight = int.Parse(val, CultureInfo.InvariantCulture); } break; case 14: request.StartTimeTicks = long.Parse(val, CultureInfo.InvariantCulture); break; case 15: if (videoRequest is not null) { videoRequest.Level = val; } break; case 16: if (videoRequest is not null) { videoRequest.MaxRefFrames = int.Parse(val, CultureInfo.InvariantCulture); } break; case 17: if (videoRequest is not null) { videoRequest.MaxVideoBitDepth = int.Parse(val, CultureInfo.InvariantCulture); } break; case 18: if (videoRequest is not null) { videoRequest.Profile = val; } break; case 19: // cabac no longer used break; case 20: request.PlaySessionId = val; break; case 21: // api_key break; case 22: request.LiveStreamId = val; break; case 23: // Duplicating ItemId because of MediaMonkey break; case 24: if (videoRequest is not null) { videoRequest.CopyTimestamps = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); } break; case 25: if (!string.IsNullOrWhiteSpace(val) && videoRequest is not null) { if (Enum.TryParse(val, out SubtitleDeliveryMethod method)) { videoRequest.SubtitleMethod = method; } } break; case 26: request.TranscodingMaxAudioChannels = int.Parse(val, CultureInfo.InvariantCulture); break; case 27: if (videoRequest is not null) { videoRequest.EnableSubtitlesInManifest = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); } break; case 28: request.Tag = val; break; case 29: if (videoRequest is not null) { videoRequest.RequireAvc = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); } break; case 30: request.SubtitleCodec = val; break; case 31: if (videoRequest is not null) { videoRequest.RequireNonAnamorphic = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); } break; case 32: if (videoRequest is not null) { videoRequest.DeInterlace = string.Equals("true", val, StringComparison.OrdinalIgnoreCase); } break; case 33: request.TranscodeReasons = val; break; } } } /// /// Parses the container into its file extension. /// /// The container. private static string? GetContainerFileExtension(string? container) { if (string.Equals(container, "mpegts", StringComparison.OrdinalIgnoreCase)) { return "ts"; } if (string.Equals(container, "matroska", StringComparison.OrdinalIgnoreCase)) { return "mkv"; } return container; } }