jellyfin/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs
2024-09-07 11:23:48 -06:00

626 lines
25 KiB
C#

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AsyncKeyedLock;
using Jellyfin.Data.Entities;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Drawing;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.MediaEncoding;
using MediaBrowser.Controller.Trickplay;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.IO;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace Jellyfin.Server.Implementations.Trickplay;
/// <summary>
/// ITrickplayManager implementation.
/// </summary>
public class TrickplayManager : ITrickplayManager
{
private readonly ILogger<TrickplayManager> _logger;
private readonly IMediaEncoder _mediaEncoder;
private readonly IFileSystem _fileSystem;
private readonly EncodingHelper _encodingHelper;
private readonly ILibraryManager _libraryManager;
private readonly IServerConfigurationManager _config;
private readonly IImageEncoder _imageEncoder;
private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
private readonly IApplicationPaths _appPaths;
private static readonly AsyncNonKeyedLocker _resourcePool = new(1);
private static readonly string[] _trickplayImgExtensions = { ".jpg" };
/// <summary>
/// Initializes a new instance of the <see cref="TrickplayManager"/> class.
/// </summary>
/// <param name="logger">The logger.</param>
/// <param name="mediaEncoder">The media encoder.</param>
/// <param name="fileSystem">The file systen.</param>
/// <param name="encodingHelper">The encoding helper.</param>
/// <param name="libraryManager">The library manager.</param>
/// <param name="config">The server configuration manager.</param>
/// <param name="imageEncoder">The image encoder.</param>
/// <param name="dbProvider">The database provider.</param>
/// <param name="appPaths">The application paths.</param>
public TrickplayManager(
ILogger<TrickplayManager> logger,
IMediaEncoder mediaEncoder,
IFileSystem fileSystem,
EncodingHelper encodingHelper,
ILibraryManager libraryManager,
IServerConfigurationManager config,
IImageEncoder imageEncoder,
IDbContextFactory<JellyfinDbContext> dbProvider,
IApplicationPaths appPaths)
{
_logger = logger;
_mediaEncoder = mediaEncoder;
_fileSystem = fileSystem;
_encodingHelper = encodingHelper;
_libraryManager = libraryManager;
_config = config;
_imageEncoder = imageEncoder;
_dbProvider = dbProvider;
_appPaths = appPaths;
}
/// <inheritdoc />
public async Task MoveGeneratedTrickplayDataAsync(Video video, LibraryOptions? libraryOptions, CancellationToken cancellationToken)
{
var options = _config.Configuration.TrickplayOptions;
if (!CanGenerateTrickplay(video, options.Interval))
{
return;
}
var existingTrickplayResolutions = await GetTrickplayResolutions(video.Id).ConfigureAwait(false);
foreach (var resolution in existingTrickplayResolutions)
{
cancellationToken.ThrowIfCancellationRequested();
var existingResolution = resolution.Key;
var tileWidth = resolution.Value.TileWidth;
var tileHeight = resolution.Value.TileHeight;
var shouldBeSavedWithMedia = libraryOptions is null ? false : libraryOptions.SaveTrickplayWithMedia;
var localOutputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, existingResolution, false);
var mediaOutputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, existingResolution, true);
if (shouldBeSavedWithMedia && Directory.Exists(localOutputDir))
{
var localDirFiles = Directory.GetFiles(localOutputDir);
var mediaDirExists = Directory.Exists(mediaOutputDir);
if (localDirFiles.Length > 0 && ((mediaDirExists && Directory.GetFiles(mediaOutputDir).Length == 0) || !mediaDirExists))
{
// Move images from local dir to media dir
MoveContent(localOutputDir, mediaOutputDir);
_logger.LogInformation("Moved trickplay images for {ItemName} to {Location}", video.Name, mediaOutputDir);
}
}
else if (Directory.Exists(mediaOutputDir))
{
var mediaDirFiles = Directory.GetFiles(mediaOutputDir);
var localDirExists = Directory.Exists(localOutputDir);
if (mediaDirFiles.Length > 0 && ((localDirExists && Directory.GetFiles(localOutputDir).Length == 0) || !localDirExists))
{
// Move images from media dir to local dir
MoveContent(mediaOutputDir, localOutputDir);
_logger.LogInformation("Moved trickplay images for {ItemName} to {Location}", video.Name, localOutputDir);
}
}
}
}
private void MoveContent(string sourceFolder, string destinationFolder)
{
_fileSystem.MoveDirectory(sourceFolder, destinationFolder);
var parent = Directory.GetParent(sourceFolder);
if (parent is not null)
{
var parentContent = Directory.GetDirectories(parent.FullName);
if (parentContent.Length == 0)
{
Directory.Delete(parent.FullName);
}
}
}
/// <inheritdoc />
public async Task RefreshTrickplayDataAsync(Video video, bool replace, LibraryOptions? libraryOptions, CancellationToken cancellationToken)
{
_logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace);
var options = _config.Configuration.TrickplayOptions;
if (options.Interval < 1000)
{
_logger.LogWarning("Trickplay image interval {Interval} is too small, reset to the minimum valid value of 1000", options.Interval);
options.Interval = 1000;
}
foreach (var width in options.WidthResolutions)
{
cancellationToken.ThrowIfCancellationRequested();
await RefreshTrickplayDataInternal(
video,
replace,
width,
options,
libraryOptions,
cancellationToken).ConfigureAwait(false);
}
}
private async Task RefreshTrickplayDataInternal(
Video video,
bool replace,
int width,
TrickplayOptions options,
LibraryOptions? libraryOptions,
CancellationToken cancellationToken)
{
if (!CanGenerateTrickplay(video, options.Interval))
{
return;
}
var imgTempDir = string.Empty;
using (await _resourcePool.LockAsync(cancellationToken).ConfigureAwait(false))
{
try
{
// Extract images
// Note: Media sources under parent items exist as their own video/item as well. Only use this video stream for trickplay.
var mediaSource = video.GetMediaSources(false).Find(source => Guid.Parse(source.Id).Equals(video.Id));
if (mediaSource is null)
{
_logger.LogDebug("Found no matching media source for item {ItemId}", video.Id);
return;
}
var mediaPath = mediaSource.Path;
if (!File.Exists(mediaPath))
{
_logger.LogWarning("Media not found at {Path} for item {ItemID}", mediaPath, video.Id);
return;
}
// The width has to be even, otherwise a lot of filters will not be able to sample it
var actualWidth = 2 * (width / 2);
// Force using the video width when the trickplay setting has a too large width
if (mediaSource.VideoStream.Width is not null && mediaSource.VideoStream.Width < width)
{
_logger.LogWarning("Video width {VideoWidth} is smaller than trickplay setting {TrickPlayWidth}, using video width for thumbnails", mediaSource.VideoStream.Width, width);
actualWidth = 2 * ((int)mediaSource.VideoStream.Width / 2);
}
var tileWidth = options.TileWidth;
var tileHeight = options.TileHeight;
var saveWithMedia = libraryOptions is null ? false : libraryOptions.SaveTrickplayWithMedia;
var outputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, actualWidth, saveWithMedia);
// Import existing trickplay tiles
if (!replace && Directory.Exists(outputDir))
{
var existingFiles = Directory.GetFiles(outputDir);
if (existingFiles.Length > 0)
{
var hasTrickplayResolution = await HasTrickplayResolutionAsync(video.Id, actualWidth).ConfigureAwait(false);
if (hasTrickplayResolution)
{
_logger.LogDebug("Found existing trickplay files for {ItemId}.", video.Id);
return;
}
// Import tiles
var localTrickplayInfo = new TrickplayInfo
{
ItemId = video.Id,
Width = width,
Interval = options.Interval,
TileWidth = options.TileWidth,
TileHeight = options.TileHeight,
ThumbnailCount = existingFiles.Length,
Height = 0,
Bandwidth = 0
};
foreach (var tile in existingFiles)
{
var image = _imageEncoder.GetImageSize(tile);
localTrickplayInfo.Height = Math.Max(localTrickplayInfo.Height, image.Height);
var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tile).Length * 8 / localTrickplayInfo.TileWidth / localTrickplayInfo.TileHeight / (localTrickplayInfo.Interval / 1000));
localTrickplayInfo.Bandwidth = Math.Max(localTrickplayInfo.Bandwidth, bitrate);
}
await SaveTrickplayInfo(localTrickplayInfo).ConfigureAwait(false);
_logger.LogDebug("Imported existing trickplay files for {ItemId}.", video.Id);
return;
}
}
// Generate trickplay tiles
var mediaStream = mediaSource.VideoStream;
var container = mediaSource.Container;
_logger.LogInformation("Creating trickplay files at {Width} width, for {Path} [ID: {ItemId}]", actualWidth, mediaPath, video.Id);
imgTempDir = await _mediaEncoder.ExtractVideoImagesOnIntervalAccelerated(
mediaPath,
container,
mediaSource,
mediaStream,
actualWidth,
TimeSpan.FromMilliseconds(options.Interval),
options.EnableHwAcceleration,
options.EnableHwEncoding,
options.ProcessThreads,
options.Qscale,
options.ProcessPriority,
options.EnableKeyFrameOnlyExtraction,
_encodingHelper,
cancellationToken).ConfigureAwait(false);
if (string.IsNullOrEmpty(imgTempDir) || !Directory.Exists(imgTempDir))
{
throw new InvalidOperationException("Null or invalid directory from media encoder.");
}
var images = _fileSystem.GetFiles(imgTempDir, _trickplayImgExtensions, false, false)
.Select(i => i.FullName)
.OrderBy(i => i)
.ToList();
// Create tiles
var trickplayInfo = CreateTiles(images, actualWidth, options, outputDir);
// Save tiles info
try
{
if (trickplayInfo is not null)
{
trickplayInfo.ItemId = video.Id;
await SaveTrickplayInfo(trickplayInfo).ConfigureAwait(false);
_logger.LogInformation("Finished creation of trickplay files for {0}", mediaPath);
}
else
{
throw new InvalidOperationException("Null trickplay tiles info from CreateTiles.");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while saving trickplay tiles info.");
// Make sure no files stay in metadata folders on failure
// if tiles info wasn't saved.
Directory.Delete(outputDir, true);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error creating trickplay images.");
}
finally
{
if (!string.IsNullOrEmpty(imgTempDir))
{
Directory.Delete(imgTempDir, true);
}
}
}
}
/// <inheritdoc />
public TrickplayInfo CreateTiles(IReadOnlyList<string> images, int width, TrickplayOptions options, string outputDir)
{
if (images.Count == 0)
{
throw new ArgumentException("Can't create trickplay from 0 images.");
}
var workDir = Path.Combine(_appPaths.TempDirectory, "trickplay_" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(workDir);
var trickplayInfo = new TrickplayInfo
{
Width = width,
Interval = options.Interval,
TileWidth = options.TileWidth,
TileHeight = options.TileHeight,
ThumbnailCount = images.Count,
// Set during image generation
Height = 0,
Bandwidth = 0
};
/*
* Generate trickplay tiles from sets of thumbnails
*/
var imageOptions = new ImageCollageOptions
{
Width = trickplayInfo.TileWidth,
Height = trickplayInfo.TileHeight
};
var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight;
var requiredTiles = (int)Math.Ceiling((double)images.Count / thumbnailsPerTile);
for (int i = 0; i < requiredTiles; i++)
{
// Set output/input paths
var tilePath = Path.Combine(workDir, $"{i}.jpg");
imageOptions.OutputPath = tilePath;
imageOptions.InputPaths = images.Skip(i * thumbnailsPerTile).Take(Math.Min(thumbnailsPerTile, images.Count - (i * thumbnailsPerTile))).ToList();
// Generate image and use returned height for tiles info
var height = _imageEncoder.CreateTrickplayTile(imageOptions, options.JpegQuality, trickplayInfo.Width, trickplayInfo.Height != 0 ? trickplayInfo.Height : null);
if (trickplayInfo.Height == 0)
{
trickplayInfo.Height = height;
}
// Update bitrate
var bitrate = (int)Math.Ceiling(new FileInfo(tilePath).Length * 8m / trickplayInfo.TileWidth / trickplayInfo.TileHeight / (trickplayInfo.Interval / 1000m));
trickplayInfo.Bandwidth = Math.Max(trickplayInfo.Bandwidth, bitrate);
}
/*
* Move trickplay tiles to output directory
*/
Directory.CreateDirectory(Directory.GetParent(outputDir)!.FullName);
// Replace existing tiles if they already exist
if (Directory.Exists(outputDir))
{
Directory.Delete(outputDir, true);
}
_fileSystem.MoveDirectory(workDir, outputDir);
return trickplayInfo;
}
private bool CanGenerateTrickplay(Video video, int interval)
{
var videoType = video.VideoType;
if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay)
{
return false;
}
if (video.IsPlaceHolder)
{
return false;
}
if (video.IsShortcut)
{
return false;
}
if (!video.IsCompleteMedia)
{
return false;
}
if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks)
{
return false;
}
var libraryOptions = _libraryManager.GetLibraryOptions(video);
if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction)
{
return false;
}
// Can't extract images if there are no video streams
return video.GetMediaStreams().Count > 0;
}
/// <inheritdoc />
public async Task<Dictionary<int, TrickplayInfo>> GetTrickplayResolutions(Guid itemId)
{
var trickplayResolutions = new Dictionary<int, TrickplayInfo>();
var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
await using (dbContext.ConfigureAwait(false))
{
var trickplayInfos = await dbContext.TrickplayInfos
.AsNoTracking()
.Where(i => i.ItemId.Equals(itemId))
.ToListAsync()
.ConfigureAwait(false);
foreach (var info in trickplayInfos)
{
trickplayResolutions[info.Width] = info;
}
}
return trickplayResolutions;
}
/// <inheritdoc />
public async Task<IReadOnlyList<Guid>> GetTrickplayItemsAsync()
{
List<Guid> trickplayItems;
var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
await using (dbContext.ConfigureAwait(false))
{
trickplayItems = await dbContext.TrickplayInfos
.AsNoTracking()
.Select(i => i.ItemId)
.ToListAsync()
.ConfigureAwait(false);
}
return trickplayItems;
}
/// <inheritdoc />
public async Task SaveTrickplayInfo(TrickplayInfo info)
{
var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
await using (dbContext.ConfigureAwait(false))
{
var oldInfo = await dbContext.TrickplayInfos.FindAsync(info.ItemId, info.Width).ConfigureAwait(false);
if (oldInfo is not null)
{
dbContext.TrickplayInfos.Remove(oldInfo);
}
dbContext.Add(info);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
}
}
/// <inheritdoc />
public async Task<Dictionary<string, Dictionary<int, TrickplayInfo>>> GetTrickplayManifest(BaseItem item)
{
var trickplayManifest = new Dictionary<string, Dictionary<int, TrickplayInfo>>();
foreach (var mediaSource in item.GetMediaSources(false))
{
var mediaSourceId = Guid.Parse(mediaSource.Id);
var trickplayResolutions = await GetTrickplayResolutions(mediaSourceId).ConfigureAwait(false);
if (trickplayResolutions.Count > 0)
{
trickplayManifest[mediaSource.Id] = trickplayResolutions;
}
}
return trickplayManifest;
}
/// <inheritdoc />
public async Task<string> GetTrickplayTilePathAsync(BaseItem item, int width, int index, bool saveWithMedia)
{
var trickplayResolutions = await GetTrickplayResolutions(item.Id).ConfigureAwait(false);
if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo))
{
return Path.Combine(GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, width, saveWithMedia), index + ".jpg");
}
return string.Empty;
}
/// <inheritdoc />
public async Task<string?> GetHlsPlaylist(Guid itemId, int width, string? apiKey)
{
var trickplayResolutions = await GetTrickplayResolutions(itemId).ConfigureAwait(false);
if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo))
{
var builder = new StringBuilder(128);
if (trickplayInfo.ThumbnailCount > 0)
{
const string urlFormat = "{0}.jpg?MediaSourceId={1}&api_key={2}";
const string decimalFormat = "{0:0.###}";
var resolution = $"{trickplayInfo.Width}x{trickplayInfo.Height}";
var layout = $"{trickplayInfo.TileWidth}x{trickplayInfo.TileHeight}";
var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight;
var thumbnailDuration = trickplayInfo.Interval / 1000d;
var infDuration = thumbnailDuration * thumbnailsPerTile;
var tileCount = (int)Math.Ceiling((decimal)trickplayInfo.ThumbnailCount / thumbnailsPerTile);
builder
.AppendLine("#EXTM3U")
.Append("#EXT-X-TARGETDURATION:")
.AppendLine(tileCount.ToString(CultureInfo.InvariantCulture))
.AppendLine("#EXT-X-VERSION:7")
.AppendLine("#EXT-X-MEDIA-SEQUENCE:1")
.AppendLine("#EXT-X-PLAYLIST-TYPE:VOD")
.AppendLine("#EXT-X-IMAGES-ONLY");
for (int i = 0; i < tileCount; i++)
{
// All tiles prior to the last must contain full amount of thumbnails (no black).
if (i == tileCount - 1)
{
thumbnailsPerTile = trickplayInfo.ThumbnailCount - (i * thumbnailsPerTile);
infDuration = thumbnailDuration * thumbnailsPerTile;
}
// EXTINF
builder
.Append("#EXTINF:")
.AppendFormat(CultureInfo.InvariantCulture, decimalFormat, infDuration)
.AppendLine(",");
// EXT-X-TILES
builder
.Append("#EXT-X-TILES:RESOLUTION=")
.Append(resolution)
.Append(",LAYOUT=")
.Append(layout)
.Append(",DURATION=")
.AppendFormat(CultureInfo.InvariantCulture, decimalFormat, thumbnailDuration)
.AppendLine();
// URL
builder
.AppendFormat(
CultureInfo.InvariantCulture,
urlFormat,
i.ToString(CultureInfo.InvariantCulture),
itemId.ToString("N"),
apiKey)
.AppendLine();
}
builder.AppendLine("#EXT-X-ENDLIST");
return builder.ToString();
}
}
return null;
}
/// <inheritdoc />
public string GetTrickplayDirectory(BaseItem item, int tileWidth, int tileHeight, int width, bool saveWithMedia = false)
{
var path = saveWithMedia
? Path.Combine(item.ContainingFolderPath, Path.ChangeExtension(item.Path, ".trickplay"))
: Path.Combine(item.GetInternalMetadataPath(), "trickplay");
var subdirectory = string.Format(
CultureInfo.InvariantCulture,
"{0} - {1}x{2}",
width.ToString(CultureInfo.InvariantCulture),
tileWidth.ToString(CultureInfo.InvariantCulture),
tileHeight.ToString(CultureInfo.InvariantCulture));
return Path.Combine(path, subdirectory);
}
private async Task<bool> HasTrickplayResolutionAsync(Guid itemId, int width)
{
var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
await using (dbContext.ConfigureAwait(false))
{
return await dbContext.TrickplayInfos
.AsNoTracking()
.Where(i => i.ItemId.Equals(itemId))
.AnyAsync(i => i.Width == width)
.ConfigureAwait(false);
}
}
}