using System; using System.Collections.Generic; using System.Globalization; using System.IO; using BlurHashSharp.SkiaSharp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Extensions; using MediaBrowser.Model.Drawing; using Microsoft.Extensions.Logging; using SkiaSharp; using static Jellyfin.Drawing.Skia.SkiaHelper; namespace Jellyfin.Drawing.Skia { /// /// Image encoder that uses to manipulate images. /// public class SkiaEncoder : IImageEncoder { private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" }; private readonly ILogger _logger; private readonly IApplicationPaths _appPaths; /// /// Initializes a new instance of the class. /// /// The application logger. /// The application paths. public SkiaEncoder(ILogger logger, IApplicationPaths appPaths) { _logger = logger; _appPaths = appPaths; } /// public string Name => "Skia"; /// public bool SupportsImageCollageCreation => true; /// public bool SupportsImageEncoding => true; /// public IReadOnlyCollection SupportedInputFormats => new HashSet(StringComparer.OrdinalIgnoreCase) { "jpeg", "jpg", "png", "dng", "webp", "gif", "bmp", "ico", "astc", "ktx", "pkm", "wbmp", // TODO: check if these are supported on multiple platforms // https://github.com/google/skia/blob/master/infra/bots/recipes/test.py#L454 // working on windows at least "cr2", "nef", "arw" }; /// public IReadOnlyCollection SupportedOutputFormats => new HashSet() { ImageFormat.Webp, ImageFormat.Jpg, ImageFormat.Png }; /// /// Check if the native lib is available. /// /// True if the native lib is available, otherwise false. public static bool IsNativeLibAvailable() { try { // test an operation that requires the native library SKPMColor.PreMultiply(SKColors.Black); return true; } catch (Exception) { return false; } } private static bool IsTransparent(SKColor color) => (color.Red == 255 && color.Green == 255 && color.Blue == 255) || color.Alpha == 0; /// /// Convert a to a . /// /// The format to convert. /// The converted format. public static SKEncodedImageFormat GetImageFormat(ImageFormat selectedFormat) { return selectedFormat switch { ImageFormat.Bmp => SKEncodedImageFormat.Bmp, ImageFormat.Jpg => SKEncodedImageFormat.Jpeg, ImageFormat.Gif => SKEncodedImageFormat.Gif, ImageFormat.Webp => SKEncodedImageFormat.Webp, _ => SKEncodedImageFormat.Png }; } private static bool IsTransparentRow(SKBitmap bmp, int row) { for (var i = 0; i < bmp.Width; ++i) { if (!IsTransparent(bmp.GetPixel(i, row))) { return false; } } return true; } private static bool IsTransparentColumn(SKBitmap bmp, int col) { for (var i = 0; i < bmp.Height; ++i) { if (!IsTransparent(bmp.GetPixel(col, i))) { return false; } } return true; } private SKBitmap CropWhiteSpace(SKBitmap bitmap) { var topmost = 0; while (topmost < bitmap.Height && IsTransparentRow(bitmap, topmost)) { topmost++; } int bottommost = bitmap.Height; while (bottommost >= 0 && IsTransparentRow(bitmap, bottommost - 1)) { bottommost--; } var leftmost = 0; while (leftmost < bitmap.Width && IsTransparentColumn(bitmap, leftmost)) { leftmost++; } var rightmost = bitmap.Width; while (rightmost >= 0 && IsTransparentColumn(bitmap, rightmost - 1)) { rightmost--; } var newRect = SKRectI.Create(leftmost, topmost, rightmost - leftmost, bottommost - topmost); using var image = SKImage.FromBitmap(bitmap); using var subset = image.Subset(newRect); return SKBitmap.FromImage(subset); } /// /// The path is null. /// The path is not valid. /// The file at the specified path could not be used to generate a codec. public ImageDimensions GetImageSize(string path) { if (!File.Exists(path)) { throw new FileNotFoundException("File not found", path); } using var codec = SKCodec.Create(path, out SKCodecResult result); EnsureSuccess(result); var info = codec.Info; return new ImageDimensions(info.Width, info.Height); } /// /// The path is null. /// The path is not valid. /// The file at the specified path could not be used to generate a codec. public string GetImageBlurHash(int xComp, int yComp, string path) { if (path == null) { throw new ArgumentNullException(nameof(path)); } // Any larger than 128x128 is too slow and there's no visually discernible difference return BlurHashEncoder.Encode(xComp, yComp, path, 128, 128); } private static bool HasDiacritics(string text) => !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal); private bool RequiresSpecialCharacterHack(string path) { for (int i = 0; i < path.Length; i++) { if (char.GetUnicodeCategory(path[i]) == UnicodeCategory.OtherLetter) { return true; } } return HasDiacritics(path); } private string NormalizePath(string path) { if (!RequiresSpecialCharacterHack(path)) { return path; } var tempPath = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + Path.GetExtension(path)); Directory.CreateDirectory(Path.GetDirectoryName(tempPath)); File.Copy(path, tempPath, true); return tempPath; } private static SKEncodedOrigin GetSKEncodedOrigin(ImageOrientation? orientation) { if (!orientation.HasValue) { return SKEncodedOrigin.TopLeft; } return orientation.Value switch { ImageOrientation.TopRight => SKEncodedOrigin.TopRight, ImageOrientation.RightTop => SKEncodedOrigin.RightTop, ImageOrientation.RightBottom => SKEncodedOrigin.RightBottom, ImageOrientation.LeftTop => SKEncodedOrigin.LeftTop, ImageOrientation.LeftBottom => SKEncodedOrigin.LeftBottom, ImageOrientation.BottomRight => SKEncodedOrigin.BottomRight, ImageOrientation.BottomLeft => SKEncodedOrigin.BottomLeft, _ => SKEncodedOrigin.TopLeft }; } /// /// Decode an image. /// /// The filepath of the image to decode. /// Whether to force clean the bitmap. /// The orientation of the image. /// The detected origin of the image. /// The resulting bitmap of the image. internal SKBitmap? Decode(string path, bool forceCleanBitmap, ImageOrientation? orientation, out SKEncodedOrigin origin) { if (!File.Exists(path)) { throw new FileNotFoundException("File not found", path); } var requiresTransparencyHack = _transparentImageTypes.Contains(Path.GetExtension(path)); if (requiresTransparencyHack || forceCleanBitmap) { using var codec = SKCodec.Create(NormalizePath(path)); if (codec == null) { origin = GetSKEncodedOrigin(orientation); return null; } // create the bitmap var bitmap = new SKBitmap(codec.Info.Width, codec.Info.Height, !requiresTransparencyHack); // decode _ = codec.GetPixels(bitmap.Info, bitmap.GetPixels()); origin = codec.EncodedOrigin; return bitmap; } var resultBitmap = SKBitmap.Decode(NormalizePath(path)); if (resultBitmap == null) { return Decode(path, true, orientation, out origin); } // If we have to resize these they often end up distorted if (resultBitmap.ColorType == SKColorType.Gray8) { using (resultBitmap) { return Decode(path, true, orientation, out origin); } } origin = SKEncodedOrigin.TopLeft; return resultBitmap; } private SKBitmap? GetBitmap(string path, bool cropWhitespace, bool forceAnalyzeBitmap, ImageOrientation? orientation, out SKEncodedOrigin origin) { if (cropWhitespace) { using var bitmap = Decode(path, forceAnalyzeBitmap, orientation, out origin); return bitmap == null ? null : CropWhiteSpace(bitmap); } return Decode(path, forceAnalyzeBitmap, orientation, out origin); } private SKBitmap? GetBitmap(string path, bool cropWhitespace, bool autoOrient, ImageOrientation? orientation) { if (autoOrient) { var bitmap = GetBitmap(path, cropWhitespace, true, orientation, out var origin); if (bitmap != null && origin != SKEncodedOrigin.TopLeft) { using (bitmap) { return OrientImage(bitmap, origin); } } return bitmap; } return GetBitmap(path, cropWhitespace, false, orientation, out _); } private SKBitmap OrientImage(SKBitmap bitmap, SKEncodedOrigin origin) { if (origin == SKEncodedOrigin.Default) { return bitmap; } var needsFlip = origin == SKEncodedOrigin.LeftBottom || origin == SKEncodedOrigin.LeftTop || origin == SKEncodedOrigin.RightBottom || origin == SKEncodedOrigin.RightTop; var rotated = needsFlip ? new SKBitmap(bitmap.Height, bitmap.Width) : new SKBitmap(bitmap.Width, bitmap.Height); using var surface = new SKCanvas(rotated); var midX = (float)rotated.Width / 2; var midY = (float)rotated.Height / 2; switch (origin) { case SKEncodedOrigin.TopRight: surface.Scale(-1, 1, midX, midY); break; case SKEncodedOrigin.BottomRight: surface.RotateDegrees(180, midX, midY); break; case SKEncodedOrigin.BottomLeft: surface.Scale(1, -1, midX, midY); break; case SKEncodedOrigin.LeftTop: surface.Translate(0, -rotated.Height); surface.Scale(1, -1, midX, midY); surface.RotateDegrees(-90); break; case SKEncodedOrigin.RightTop: surface.Translate(rotated.Width, 0); surface.RotateDegrees(90); break; case SKEncodedOrigin.RightBottom: surface.Translate(rotated.Width, 0); surface.Scale(1, -1, midX, midY); surface.RotateDegrees(90); break; case SKEncodedOrigin.LeftBottom: surface.Translate(0, rotated.Height); surface.RotateDegrees(-90); break; } surface.DrawBitmap(bitmap, 0, 0); return rotated; } /// /// Resizes an image on the CPU, by utilizing a surface and canvas. /// /// The convolutional matrix kernel used in this resize function gives a (light) sharpening effect. /// This technique is similar to effect that can be created using for example the [Convolution matrix filter in GIMP](https://docs.gimp.org/2.10/en/gimp-filter-convolution-matrix.html). /// /// The source bitmap. /// This specifies the target size and other information required to create the surface. /// This enables anti-aliasing on the SKPaint instance. /// This enables dithering on the SKPaint instance. /// The resized image. internal static SKImage ResizeImage(SKBitmap source, SKImageInfo targetInfo, bool isAntialias = false, bool isDither = false) { using var surface = SKSurface.Create(targetInfo); using var canvas = surface.Canvas; using var paint = new SKPaint(); paint.FilterQuality = SKFilterQuality.High; paint.IsAntialias = isAntialias; paint.IsDither = isDither; var kernel = new float[9] { 0, -.1f, 0, -.1f, 1.4f, -.1f, 0, -.1f, 0, }; var kernelSize = new SKSizeI(3, 3); var kernelOffset = new SKPointI(1, 1); paint.ImageFilter = SKImageFilter.CreateMatrixConvolution( kernelSize, kernel, 1f, 0f, kernelOffset, SKShaderTileMode.Clamp, false); canvas.DrawBitmap(source, SKRect.Create(0, 0, source.Width, source.Height), SKRect.Create(0, 0, targetInfo.Width, targetInfo.Height), paint); return surface.Snapshot(); } /// public string EncodeImage(string inputPath, DateTime dateModified, string outputPath, bool autoOrient, ImageOrientation? orientation, int quality, ImageProcessingOptions options, ImageFormat selectedOutputFormat) { if (inputPath.Length == 0) { throw new ArgumentException("String can't be empty.", nameof(inputPath)); } if (outputPath.Length == 0) { throw new ArgumentException("String can't be empty.", nameof(outputPath)); } var skiaOutputFormat = GetImageFormat(selectedOutputFormat); var hasBackgroundColor = !string.IsNullOrWhiteSpace(options.BackgroundColor); var hasForegroundColor = !string.IsNullOrWhiteSpace(options.ForegroundLayer); var blur = options.Blur ?? 0; var hasIndicator = options.AddPlayedIndicator || options.UnplayedCount.HasValue || !options.PercentPlayed.Equals(0); using var bitmap = GetBitmap(inputPath, options.CropWhiteSpace, autoOrient, orientation); if (bitmap == null) { throw new InvalidDataException($"Skia unable to read image {inputPath}"); } var originalImageSize = new ImageDimensions(bitmap.Width, bitmap.Height); if (!options.CropWhiteSpace && options.HasDefaultOptions(inputPath, originalImageSize) && !autoOrient) { // Just spit out the original file if all the options are default return inputPath; } var newImageSize = ImageHelper.GetNewImageSize(options, originalImageSize); var width = newImageSize.Width; var height = newImageSize.Height; // scale image (the FromImage creates a copy) using var resizedBitmap = SKBitmap.FromImage(ResizeImage(bitmap, new SKImageInfo(width, height, bitmap.ColorType, bitmap.AlphaType, bitmap.ColorSpace))); // If all we're doing is resizing then we can stop now if (!hasBackgroundColor && !hasForegroundColor && blur == 0 && !hasIndicator) { Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); using var outputStream = new SKFileWStream(outputPath); using var pixmap = new SKPixmap(new SKImageInfo(width, height), resizedBitmap.GetPixels()); resizedBitmap.Encode(outputStream, skiaOutputFormat, quality); return outputPath; } // create bitmap to use for canvas drawing used to draw into bitmap using var saveBitmap = new SKBitmap(width, height); using var canvas = new SKCanvas(saveBitmap); // set background color if present if (hasBackgroundColor) { canvas.Clear(SKColor.Parse(options.BackgroundColor)); } // Add blur if option is present if (blur > 0) { // create image from resized bitmap to apply blur using var paint = new SKPaint(); using var filter = SKImageFilter.CreateBlur(blur, blur); paint.ImageFilter = filter; canvas.DrawBitmap(resizedBitmap, SKRect.Create(width, height), paint); } else { // draw resized bitmap onto canvas canvas.DrawBitmap(resizedBitmap, SKRect.Create(width, height)); } // If foreground layer present then draw if (hasForegroundColor) { if (!double.TryParse(options.ForegroundLayer, out double opacity)) { opacity = .4; } canvas.DrawColor(new SKColor(0, 0, 0, (byte)((1 - opacity) * 0xFF)), SKBlendMode.SrcOver); } if (hasIndicator) { DrawIndicator(canvas, width, height, options); } Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); using (var outputStream = new SKFileWStream(outputPath)) { using (var pixmap = new SKPixmap(new SKImageInfo(width, height), saveBitmap.GetPixels())) { pixmap.Encode(outputStream, skiaOutputFormat, quality); } } return outputPath; } /// public void CreateImageCollage(ImageCollageOptions options) { double ratio = (double)options.Width / options.Height; if (ratio >= 1.4) { new StripCollageBuilder(this).BuildThumbCollage(options.InputPaths, options.OutputPath, options.Width, options.Height); } else if (ratio >= .9) { new StripCollageBuilder(this).BuildSquareCollage(options.InputPaths, options.OutputPath, options.Width, options.Height); } else { // TODO: Create Poster collage capability new StripCollageBuilder(this).BuildSquareCollage(options.InputPaths, options.OutputPath, options.Width, options.Height); } } private void DrawIndicator(SKCanvas canvas, int imageWidth, int imageHeight, ImageProcessingOptions options) { try { var currentImageSize = new ImageDimensions(imageWidth, imageHeight); if (options.AddPlayedIndicator) { PlayedIndicatorDrawer.DrawPlayedIndicator(canvas, currentImageSize); } else if (options.UnplayedCount.HasValue) { UnplayedCountIndicator.DrawUnplayedCountIndicator(canvas, currentImageSize, options.UnplayedCount.Value); } if (options.PercentPlayed > 0) { PercentPlayedDrawer.Process(canvas, currentImageSize, options.PercentPlayed); } } catch (Exception ex) { _logger.LogError(ex, "Error drawing indicator overlay"); } } } }