using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Controller; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Drawing; using System.Drawing.Drawing2D; using System.Drawing.Imaging; using System.Globalization; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Drawing { /// /// Class ImageProcessor /// public class ImageProcessor : IImageProcessor { /// /// The us culture /// protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// /// The _cached imaged sizes /// private readonly ConcurrentDictionary _cachedImagedSizes = new ConcurrentDictionary(); /// /// Gets the list of currently registered image processors /// Image processors are specialized metadata providers that run after the normal ones /// /// The image enhancers. public IEnumerable ImageEnhancers { get; private set; } /// /// The _logger /// private readonly ILogger _logger; /// /// The _app paths /// private readonly IServerApplicationPaths _appPaths; private readonly string _imageSizeCachePath; private readonly string _croppedWhitespaceImageCachePath; private readonly string _enhancedImageCachePath; private readonly string _resizedImageCachePath; public ImageProcessor(ILogger logger, IServerApplicationPaths appPaths) { _logger = logger; _appPaths = appPaths; _imageSizeCachePath = Path.Combine(_appPaths.ImageCachePath, "image-sizes"); _croppedWhitespaceImageCachePath = Path.Combine(_appPaths.ImageCachePath, "cropped-images"); _enhancedImageCachePath = Path.Combine(_appPaths.ImageCachePath, "enhanced-images"); _resizedImageCachePath = Path.Combine(_appPaths.ImageCachePath, "resized-images"); } public void AddParts(IEnumerable enhancers) { ImageEnhancers = enhancers.ToArray(); } public async Task ProcessImage(ImageProcessingOptions options, Stream toStream) { if (options == null) { throw new ArgumentNullException("options"); } if (toStream == null) { throw new ArgumentNullException("toStream"); } var originalImagePath = options.OriginalImagePath; var dateModified = options.OriginalImageDateModified; if (options.CropWhiteSpace) { originalImagePath = await GetWhitespaceCroppedImage(originalImagePath, dateModified).ConfigureAwait(false); } // No enhancement - don't cache if (options.Enhancers.Count > 0) { var tuple = await GetEnhancedImage(originalImagePath, dateModified, options.Item, options.ImageType, options.ImageIndex, options.Enhancers).ConfigureAwait(false); originalImagePath = tuple.Item1; dateModified = tuple.Item2; } var originalImageSize = GetImageSize(originalImagePath, dateModified); // Determine the output size based on incoming parameters var newSize = DrawingUtils.Resize(originalImageSize, options.Width, options.Height, options.MaxWidth, options.MaxHeight); var quality = options.Quality ?? 90; var cacheFilePath = GetCacheFilePath(originalImagePath, newSize, quality, dateModified, options.OutputFormat, options.Indicator, options.PercentPlayed, options.BackgroundColor); try { using (var fileStream = new FileStream(cacheFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await fileStream.CopyToAsync(toStream).ConfigureAwait(false); return; } } catch (IOException) { // Cache file doesn't exist or is currently being written to } var semaphore = GetLock(cacheFilePath); await semaphore.WaitAsync().ConfigureAwait(false); // Check again in case of lock contention try { using (var fileStream = new FileStream(cacheFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await fileStream.CopyToAsync(toStream).ConfigureAwait(false); semaphore.Release(); return; } } catch (IOException) { // Cache file doesn't exist or is currently being written to } catch { semaphore.Release(); throw; } try { using (var fileStream = new FileStream(originalImagePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, true)) { // Copy to memory stream to avoid Image locking file using (var memoryStream = new MemoryStream()) { await fileStream.CopyToAsync(memoryStream).ConfigureAwait(false); using (var originalImage = Image.FromStream(memoryStream, true, false)) { var newWidth = Convert.ToInt32(newSize.Width); var newHeight = Convert.ToInt32(newSize.Height); // Graphics.FromImage will throw an exception if the PixelFormat is Indexed, so we need to handle that here using (var thumbnail = !ImageExtensions.IsPixelFormatSupportedByGraphicsObject(originalImage.PixelFormat) ? new Bitmap(originalImage, newWidth, newHeight) : new Bitmap(newWidth, newHeight, originalImage.PixelFormat)) { // Preserve the original resolution thumbnail.SetResolution(originalImage.HorizontalResolution, originalImage.VerticalResolution); using (var thumbnailGraph = Graphics.FromImage(thumbnail)) { thumbnailGraph.CompositingQuality = CompositingQuality.HighQuality; thumbnailGraph.SmoothingMode = SmoothingMode.HighQuality; thumbnailGraph.InterpolationMode = InterpolationMode.HighQualityBicubic; thumbnailGraph.PixelOffsetMode = PixelOffsetMode.HighQuality; thumbnailGraph.CompositingMode = CompositingMode.SourceOver; SetBackgroundColor(thumbnailGraph, options); thumbnailGraph.DrawImage(originalImage, 0, 0, newWidth, newHeight); DrawIndicator(thumbnailGraph, newWidth, newHeight, options.Indicator, options.PercentPlayed); var outputFormat = GetOutputFormat(originalImage, options.OutputFormat); using (var outputMemoryStream = new MemoryStream()) { // Save to the memory stream thumbnail.Save(outputFormat, outputMemoryStream, quality); var bytes = outputMemoryStream.ToArray(); await toStream.WriteAsync(bytes, 0, bytes.Length).ConfigureAwait(false); // kick off a task to cache the result CacheResizedImage(cacheFilePath, bytes, semaphore); } } } } } } } catch { semaphore.Release(); throw; } } /// /// Caches the resized image. /// /// The cache file path. /// The bytes. /// The semaphore. private void CacheResizedImage(string cacheFilePath, byte[] bytes, SemaphoreSlim semaphore) { Task.Run(async () => { try { var parentPath = Path.GetDirectoryName(cacheFilePath); Directory.CreateDirectory(parentPath); // Save to the cache location using (var cacheFileStream = new FileStream(cacheFilePath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { // Save to the filestream await cacheFileStream.WriteAsync(bytes, 0, bytes.Length).ConfigureAwait(false); } } catch (Exception ex) { _logger.ErrorException("Error writing to image cache file {0}", ex, cacheFilePath); } finally { semaphore.Release(); } }); } /// /// Sets the color of the background. /// /// The graphics. /// The options. private void SetBackgroundColor(Graphics graphics, ImageProcessingOptions options) { var color = options.BackgroundColor; if (!string.IsNullOrEmpty(color)) { Color drawingColor; try { drawingColor = ColorTranslator.FromHtml(color); } catch { drawingColor = ColorTranslator.FromHtml("#" + color); } graphics.Clear(drawingColor); } } /// /// Draws the indicator. /// /// The graphics. /// Width of the image. /// Height of the image. /// The indicator. /// The percent played. private void DrawIndicator(Graphics graphics, int imageWidth, int imageHeight, ImageOverlay? indicator, int percentPlayed) { if (!indicator.HasValue) { return; } try { if (indicator.Value == ImageOverlay.Played) { var currentImageSize = new Size(imageWidth, imageHeight); new WatchedIndicatorDrawer().Process(graphics, currentImageSize); } if (indicator.Value == ImageOverlay.PercentPlayed) { var currentImageSize = new Size(imageWidth, imageHeight); new PercentPlayedDrawer().Process(graphics, currentImageSize, percentPlayed); } } catch (Exception ex) { _logger.ErrorException("Error drawing indicator overlay", ex); } } /// /// Gets the output format. /// /// The image. /// The output format. /// ImageFormat. private ImageFormat GetOutputFormat(Image image, ImageOutputFormat outputFormat) { switch (outputFormat) { case ImageOutputFormat.Bmp: return ImageFormat.Bmp; case ImageOutputFormat.Gif: return ImageFormat.Gif; case ImageOutputFormat.Jpg: return ImageFormat.Jpeg; case ImageOutputFormat.Png: return ImageFormat.Png; default: return image.RawFormat; } } /// /// Crops whitespace from an image, caches the result, and returns the cached path /// /// The original image path. /// The date modified. /// System.String. private async Task GetWhitespaceCroppedImage(string originalImagePath, DateTime dateModified) { var name = originalImagePath; name += "datemodified=" + dateModified.Ticks; var croppedImagePath = GetCachePath(_croppedWhitespaceImageCachePath, name, Path.GetExtension(originalImagePath)); var semaphore = GetLock(croppedImagePath); await semaphore.WaitAsync().ConfigureAwait(false); // Check again in case of contention if (File.Exists(croppedImagePath)) { semaphore.Release(); return croppedImagePath; } try { using (var fileStream = new FileStream(originalImagePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, true)) { // Copy to memory stream to avoid Image locking file using (var memoryStream = new MemoryStream()) { await fileStream.CopyToAsync(memoryStream).ConfigureAwait(false); using (var originalImage = (Bitmap)Image.FromStream(memoryStream, true, false)) { var outputFormat = originalImage.RawFormat; using (var croppedImage = originalImage.CropWhitespace()) { var parentPath = Path.GetDirectoryName(croppedImagePath); Directory.CreateDirectory(parentPath); using (var outputStream = new FileStream(croppedImagePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { croppedImage.Save(outputFormat, outputStream, 100); } } } } } } catch (Exception ex) { // We have to have a catch-all here because some of the .net image methods throw a plain old Exception _logger.ErrorException("Error cropping image {0}", ex, originalImagePath); return originalImagePath; } finally { semaphore.Release(); } return croppedImagePath; } /// /// Gets the cache file path based on a set of parameters /// private string GetCacheFilePath(string originalPath, ImageSize outputSize, int quality, DateTime dateModified, ImageOutputFormat format, ImageOverlay? overlay, int percentPlayed, string backgroundColor) { var filename = originalPath; filename += "width=" + outputSize.Width; filename += "height=" + outputSize.Height; filename += "quality=" + quality; filename += "datemodified=" + dateModified.Ticks; if (format != ImageOutputFormat.Original) { filename += "f=" + format; } if (overlay.HasValue) { filename += "o=" + overlay.Value; filename += "p=" + percentPlayed; } if (!string.IsNullOrEmpty(backgroundColor)) { filename += "b=" + backgroundColor; } return GetCachePath(_resizedImageCachePath, filename, Path.GetExtension(originalPath)); } /// /// Gets the size of the image. /// /// The path. /// ImageSize. public ImageSize GetImageSize(string path) { return GetImageSize(path, File.GetLastWriteTimeUtc(path)); } /// /// Gets the size of the image. /// /// The path. /// The image date modified. /// ImageSize. /// path public ImageSize GetImageSize(string path, DateTime imageDateModified) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException("path"); } var name = path + "datemodified=" + imageDateModified.Ticks; ImageSize size; if (!_cachedImagedSizes.TryGetValue(name, out size)) { size = GetImageSizeInternal(name, path); _cachedImagedSizes.AddOrUpdate(name, size, (keyName, oldValue) => size); } return size; } /// /// Gets the image size internal. /// /// The cache key. /// The path. /// ImageSize. private ImageSize GetImageSizeInternal(string cacheKey, string path) { // Now check the file system cache var fullCachePath = GetCachePath(_imageSizeCachePath, cacheKey, ".txt"); try { var result = File.ReadAllText(fullCachePath).Split('|'); return new ImageSize { Width = double.Parse(result[0], UsCulture), Height = double.Parse(result[1], UsCulture) }; } catch (IOException) { // Cache file doesn't exist or is currently being written to } var syncLock = GetObjectLock(fullCachePath); lock (syncLock) { try { var result = File.ReadAllText(fullCachePath).Split('|'); return new ImageSize { Width = double.Parse(result[0], UsCulture), Height = double.Parse(result[1], UsCulture) }; } catch (FileNotFoundException) { // Cache file doesn't exist no biggie } catch (DirectoryNotFoundException) { // Cache file doesn't exist no biggie } var size = ImageHeader.GetDimensions(path, _logger); var parentPath = Path.GetDirectoryName(fullCachePath); Directory.CreateDirectory(parentPath); // Update the file system cache File.WriteAllText(fullCachePath, size.Width.ToString(UsCulture) + @"|" + size.Height.ToString(UsCulture)); return new ImageSize { Width = size.Width, Height = size.Height }; } } /// /// Gets the image cache tag. /// /// The item. /// Type of the image. /// The image path. /// Guid. /// item public Guid GetImageCacheTag(BaseItem item, ImageType imageType, string imagePath) { if (item == null) { throw new ArgumentNullException("item"); } if (string.IsNullOrEmpty(imagePath)) { throw new ArgumentNullException("imagePath"); } var dateModified = item.GetImageDateModified(imagePath); var supportedEnhancers = GetSupportedEnhancers(item, imageType); return GetImageCacheTag(item, imageType, imagePath, dateModified, supportedEnhancers); } /// /// Gets the image cache tag. /// /// The item. /// Type of the image. /// The original image path. /// The date modified of the original image file. /// The image enhancers. /// Guid. /// item public Guid GetImageCacheTag(BaseItem item, ImageType imageType, string originalImagePath, DateTime dateModified, IEnumerable imageEnhancers) { if (item == null) { throw new ArgumentNullException("item"); } if (imageEnhancers == null) { throw new ArgumentNullException("imageEnhancers"); } if (string.IsNullOrEmpty(originalImagePath)) { throw new ArgumentNullException("originalImagePath"); } // Cache name is created with supported enhancers combined with the last config change so we pick up new config changes var cacheKeys = imageEnhancers.Select(i => i.GetConfigurationCacheKey(item, imageType)).ToList(); cacheKeys.Add(originalImagePath + dateModified.Ticks); return string.Join("|", cacheKeys.ToArray()).GetMD5(); } /// /// Gets the enhanced image. /// /// The item. /// Type of the image. /// Index of the image. /// Task{System.String}. public async Task GetEnhancedImage(BaseItem item, ImageType imageType, int imageIndex) { var enhancers = GetSupportedEnhancers(item, imageType).ToList(); var imagePath = item.GetImagePath(imageType, imageIndex); var dateModified = item.GetImageDateModified(imagePath); var result = await GetEnhancedImage(imagePath, dateModified, item, imageType, imageIndex, enhancers); return result.Item1; } private async Task> GetEnhancedImage(string originalImagePath, DateTime dateModified, BaseItem item, ImageType imageType, int imageIndex, List enhancers) { try { // Enhance if we have enhancers var ehnancedImagePath = await GetEnhancedImageInternal(originalImagePath, dateModified, item, imageType, imageIndex, enhancers).ConfigureAwait(false); // If the path changed update dateModified if (!ehnancedImagePath.Equals(originalImagePath, StringComparison.OrdinalIgnoreCase)) { dateModified = File.GetLastWriteTimeUtc(ehnancedImagePath); return new Tuple(ehnancedImagePath, dateModified); } } catch (Exception ex) { _logger.Error("Error enhancing image", ex); } return new Tuple(originalImagePath, dateModified); } /// /// Runs an image through the image enhancers, caches the result, and returns the cached path /// /// The original image path. /// The date modified of the original image file. /// The item. /// Type of the image. /// Index of the image. /// The supported enhancers. /// System.String. /// originalImagePath private async Task GetEnhancedImageInternal(string originalImagePath, DateTime dateModified, BaseItem item, ImageType imageType, int imageIndex, List supportedEnhancers) { if (string.IsNullOrEmpty(originalImagePath)) { throw new ArgumentNullException("originalImagePath"); } if (item == null) { throw new ArgumentNullException("item"); } var cacheGuid = GetImageCacheTag(item, imageType, originalImagePath, dateModified, supportedEnhancers); // All enhanced images are saved as png to allow transparency var enhancedImagePath = GetCachePath(_enhancedImageCachePath, cacheGuid + ".png"); var semaphore = GetLock(enhancedImagePath); await semaphore.WaitAsync().ConfigureAwait(false); // Check again in case of contention if (File.Exists(enhancedImagePath)) { semaphore.Release(); return enhancedImagePath; } try { using (var fileStream = new FileStream(originalImagePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, true)) { // Copy to memory stream to avoid Image locking file using (var memoryStream = new MemoryStream()) { await fileStream.CopyToAsync(memoryStream).ConfigureAwait(false); using (var originalImage = Image.FromStream(memoryStream, true, false)) { //Pass the image through registered enhancers using (var newImage = await ExecuteImageEnhancers(supportedEnhancers, originalImage, item, imageType, imageIndex).ConfigureAwait(false)) { var parentDirectory = Path.GetDirectoryName(enhancedImagePath); Directory.CreateDirectory(parentDirectory); //And then save it in the cache using (var outputStream = new FileStream(enhancedImagePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { newImage.Save(ImageFormat.Png, outputStream, 100); } } } } } } finally { semaphore.Release(); } return enhancedImagePath; } /// /// Executes the image enhancers. /// /// The image enhancers. /// The original image. /// The item. /// Type of the image. /// Index of the image. /// Task{EnhancedImage}. private async Task ExecuteImageEnhancers(IEnumerable imageEnhancers, Image originalImage, BaseItem item, ImageType imageType, int imageIndex) { var result = originalImage; // Run the enhancers sequentially in order of priority foreach (var enhancer in imageEnhancers) { var typeName = enhancer.GetType().Name; try { result = await enhancer.EnhanceImageAsync(item, result, imageType, imageIndex).ConfigureAwait(false); } catch (Exception ex) { _logger.ErrorException("{0} failed enhancing {1}", ex, typeName, item.Name); throw; } } return result; } /// /// The _semaphoreLocks /// private readonly ConcurrentDictionary _locks = new ConcurrentDictionary(); /// /// Gets the lock. /// /// The filename. /// System.Object. private object GetObjectLock(string filename) { return _locks.GetOrAdd(filename, key => new object()); } /// /// The _semaphoreLocks /// private readonly ConcurrentDictionary _semaphoreLocks = new ConcurrentDictionary(); /// /// Gets the lock. /// /// The filename. /// System.Object. private SemaphoreSlim GetLock(string filename) { return _semaphoreLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1)); } /// /// Gets the cache path. /// /// The path. /// Name of the unique. /// The file extension. /// System.String. /// /// path /// or /// uniqueName /// or /// fileExtension /// public string GetCachePath(string path, string uniqueName, string fileExtension) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException("path"); } if (string.IsNullOrEmpty(uniqueName)) { throw new ArgumentNullException("uniqueName"); } if (string.IsNullOrEmpty(fileExtension)) { throw new ArgumentNullException("fileExtension"); } var filename = uniqueName.GetMD5() + fileExtension; return GetCachePath(path, filename); } /// /// Gets the cache path. /// /// The path. /// The filename. /// System.String. /// /// path /// or /// filename /// public string GetCachePath(string path, string filename) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException("path"); } if (string.IsNullOrEmpty(filename)) { throw new ArgumentNullException("filename"); } var prefix = filename.Substring(0, 1); path = Path.Combine(path, prefix); return Path.Combine(path, filename); } public IEnumerable GetSupportedEnhancers(BaseItem item, ImageType imageType) { return ImageEnhancers.Where(i => { try { return i.Supports(item as BaseItem, imageType); } catch (Exception ex) { _logger.ErrorException("Error in image enhancer: {0}", ex, i.GetType().Name); return false; } }).ToList(); } } }