Merge pull request #2676 from GranPC/public-pr/blurhash

Implement Blurhash generation for images
This commit is contained in:
Vasily 2020-06-02 17:25:45 +03:00 committed by GitHub
commit b9618c8c01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 339 additions and 38 deletions

12
.vscode/tasks.json vendored
View File

@ -10,6 +10,16 @@
"${workspaceFolder}/Jellyfin.Server/Jellyfin.Server.csproj"
],
"problemMatcher": "$msCompile"
},
{
"label": "api tests",
"command": "dotnet",
"type": "process",
"args": [
"test",
"${workspaceFolder}/tests/MediaBrowser.Api.Tests/MediaBrowser.Api.Tests.csproj"
],
"problemMatcher": "$msCompile"
}
]
}
}

View File

@ -313,6 +313,27 @@ namespace Emby.Drawing
public ImageDimensions GetImageDimensions(string path)
=> _imageEncoder.GetImageSize(path);
/// <inheritdoc />
public string GetImageBlurHash(string path)
{
var size = GetImageDimensions(path);
if (size.Width <= 0 || size.Height <= 0)
{
return string.Empty;
}
// We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance.
// One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width.
// See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components
float xCompF = MathF.Sqrt(16.0f * size.Width / size.Height);
float yCompF = xCompF * size.Height / size.Width;
int xComp = Math.Min((int)xCompF + 1, 9);
int yComp = Math.Min((int)yCompF + 1, 9);
return _imageEncoder.GetImageBlurHash(xComp, yComp, path);
}
/// <inheritdoc />
public string GetImageCacheTag(BaseItem item, ItemImageInfo image)
=> (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);

View File

@ -42,5 +42,11 @@ namespace Emby.Drawing
{
throw new NotImplementedException();
}
/// <inheritdoc />
public string GetImageBlurHash(int xComp, int yComp, string path)
{
throw new NotImplementedException();
}
}
}

View File

@ -1141,24 +1141,24 @@ namespace Emby.Server.Implementations.Data
public string ToValueString(ItemImageInfo image)
{
var delimeter = "*";
const string Delimeter = "*";
var path = image.Path;
if (path == null)
{
path = string.Empty;
}
var path = image.Path ?? string.Empty;
var hash = image.BlurHash ?? string.Empty;
return GetPathToSave(path) +
delimeter +
Delimeter +
image.DateModified.Ticks.ToString(CultureInfo.InvariantCulture) +
delimeter +
Delimeter +
image.Type +
delimeter +
Delimeter +
image.Width.ToString(CultureInfo.InvariantCulture) +
delimeter +
image.Height.ToString(CultureInfo.InvariantCulture);
Delimeter +
image.Height.ToString(CultureInfo.InvariantCulture) +
Delimeter +
// Replace delimiters with other characters.
// This can be removed when we migrate to a proper DB.
hash.Replace('*', '/').Replace('|', '\\');
}
public ItemImageInfo ItemImageInfoFromValueString(string value)
@ -1192,6 +1192,11 @@ namespace Emby.Server.Implementations.Data
image.Width = width;
image.Height = height;
}
if (parts.Length >= 6)
{
image.BlurHash = parts[5].Replace('/', '*').Replace('\\', '|');
}
}
return image;

View File

@ -605,7 +605,7 @@ namespace Emby.Server.Implementations.Dto
if (dictionary.TryGetValue(person.Name, out Person entity))
{
baseItemPerson.PrimaryImageTag = GetImageCacheTag(entity, ImageType.Primary);
baseItemPerson.PrimaryImageTag = GetTagAndFillBlurhash(dto, entity, ImageType.Primary);
baseItemPerson.Id = entity.Id.ToString("N", CultureInfo.InvariantCulture);
list.Add(baseItemPerson);
}
@ -654,6 +654,70 @@ namespace Emby.Server.Implementations.Dto
return _libraryManager.GetGenreId(name);
}
private string GetTagAndFillBlurhash(BaseItemDto dto, BaseItem item, ImageType imageType, int imageIndex = 0)
{
var image = item.GetImageInfo(imageType, imageIndex);
if (image != null)
{
return GetTagAndFillBlurhash(dto, item, image);
}
return null;
}
private string GetTagAndFillBlurhash(BaseItemDto dto, BaseItem item, ItemImageInfo image)
{
var tag = GetImageCacheTag(item, image);
if (!string.IsNullOrEmpty(image.BlurHash))
{
if (dto.ImageBlurHashes == null)
{
dto.ImageBlurHashes = new Dictionary<ImageType, Dictionary<string, string>>();
}
if (!dto.ImageBlurHashes.ContainsKey(image.Type))
{
dto.ImageBlurHashes[image.Type] = new Dictionary<string, string>();
}
dto.ImageBlurHashes[image.Type][tag] = image.BlurHash;
}
return tag;
}
private string[] GetTagsAndFillBlurhashes(BaseItemDto dto, BaseItem item, ImageType imageType, int limit)
{
return GetTagsAndFillBlurhashes(dto, item, imageType, item.GetImages(imageType).Take(limit).ToList());
}
private string[] GetTagsAndFillBlurhashes(BaseItemDto dto, BaseItem item, ImageType imageType, List<ItemImageInfo> images)
{
var tags = GetImageTags(item, images);
var hashes = new Dictionary<string, string>();
for (int i = 0; i < images.Count; i++)
{
var img = images[i];
if (!string.IsNullOrEmpty(img.BlurHash))
{
var tag = tags[i];
hashes[tag] = img.BlurHash;
}
}
if (hashes.Count > 0)
{
if (dto.ImageBlurHashes == null)
{
dto.ImageBlurHashes = new Dictionary<ImageType, Dictionary<string, string>>();
}
dto.ImageBlurHashes[imageType] = hashes;
}
return tags;
}
/// <summary>
/// Sets simple property values on a DTOBaseItem
/// </summary>
@ -674,8 +738,8 @@ namespace Emby.Server.Implementations.Dto
dto.LockData = item.IsLocked;
dto.ForcedSortName = item.ForcedSortName;
}
dto.Container = item.Container;
dto.Container = item.Container;
dto.EndDate = item.EndDate;
if (options.ContainsField(ItemFields.ExternalUrls))
@ -694,10 +758,12 @@ namespace Emby.Server.Implementations.Dto
dto.AspectRatio = hasAspectRatio.AspectRatio;
}
dto.ImageBlurHashes = new Dictionary<ImageType, Dictionary<string, string>>();
var backdropLimit = options.GetImageLimit(ImageType.Backdrop);
if (backdropLimit > 0)
{
dto.BackdropImageTags = GetImageTags(item, item.GetImages(ImageType.Backdrop).Take(backdropLimit).ToList());
dto.BackdropImageTags = GetTagsAndFillBlurhashes(dto, item, ImageType.Backdrop, backdropLimit);
}
if (options.ContainsField(ItemFields.ScreenshotImageTags))
@ -705,7 +771,7 @@ namespace Emby.Server.Implementations.Dto
var screenshotLimit = options.GetImageLimit(ImageType.Screenshot);
if (screenshotLimit > 0)
{
dto.ScreenshotImageTags = GetImageTags(item, item.GetImages(ImageType.Screenshot).Take(screenshotLimit).ToList());
dto.ScreenshotImageTags = GetTagsAndFillBlurhashes(dto, item, ImageType.Screenshot, screenshotLimit);
}
}
@ -721,12 +787,11 @@ namespace Emby.Server.Implementations.Dto
// Prevent implicitly captured closure
var currentItem = item;
foreach (var image in currentItem.ImageInfos.Where(i => !currentItem.AllowsMultipleImages(i.Type))
.ToList())
foreach (var image in currentItem.ImageInfos.Where(i => !currentItem.AllowsMultipleImages(i.Type)))
{
if (options.GetImageLimit(image.Type) > 0)
{
var tag = GetImageCacheTag(item, image);
var tag = GetTagAndFillBlurhash(dto, item, image);
if (tag != null)
{
@ -871,8 +936,7 @@ namespace Emby.Server.Implementations.Dto
if (albumParent != null)
{
dto.AlbumId = albumParent.Id;
dto.AlbumPrimaryImageTag = GetImageCacheTag(albumParent, ImageType.Primary);
dto.AlbumPrimaryImageTag = GetTagAndFillBlurhash(dto, albumParent, ImageType.Primary);
}
//if (options.ContainsField(ItemFields.MediaSourceCount))
@ -1099,7 +1163,7 @@ namespace Emby.Server.Implementations.Dto
episodeSeries = episodeSeries ?? episode.Series;
if (episodeSeries != null)
{
dto.SeriesPrimaryImageTag = GetImageCacheTag(episodeSeries, ImageType.Primary);
dto.SeriesPrimaryImageTag = GetTagAndFillBlurhash(dto, episodeSeries, ImageType.Primary);
}
}
@ -1145,7 +1209,7 @@ namespace Emby.Server.Implementations.Dto
series = series ?? season.Series;
if (series != null)
{
dto.SeriesPrimaryImageTag = GetImageCacheTag(series, ImageType.Primary);
dto.SeriesPrimaryImageTag = GetTagAndFillBlurhash(dto, series, ImageType.Primary);
}
}
}
@ -1275,7 +1339,7 @@ namespace Emby.Server.Implementations.Dto
if (image != null)
{
dto.ParentLogoItemId = GetDtoId(parent);
dto.ParentLogoImageTag = GetImageCacheTag(parent, image);
dto.ParentLogoImageTag = GetTagAndFillBlurhash(dto, parent, image);
}
}
if (artLimit > 0 && !(imageTags != null && imageTags.ContainsKey(ImageType.Art)) && dto.ParentArtItemId == null)
@ -1285,7 +1349,7 @@ namespace Emby.Server.Implementations.Dto
if (image != null)
{
dto.ParentArtItemId = GetDtoId(parent);
dto.ParentArtImageTag = GetImageCacheTag(parent, image);
dto.ParentArtImageTag = GetTagAndFillBlurhash(dto, parent, image);
}
}
if (thumbLimit > 0 && !(imageTags != null && imageTags.ContainsKey(ImageType.Thumb)) && (dto.ParentThumbItemId == null || parent is Series) && !(parent is ICollectionFolder) && !(parent is UserView))
@ -1295,7 +1359,7 @@ namespace Emby.Server.Implementations.Dto
if (image != null)
{
dto.ParentThumbItemId = GetDtoId(parent);
dto.ParentThumbImageTag = GetImageCacheTag(parent, image);
dto.ParentThumbImageTag = GetTagAndFillBlurhash(dto, parent, image);
}
}
if (backdropLimit > 0 && !((dto.BackdropImageTags != null && dto.BackdropImageTags.Length > 0) || (dto.ParentBackdropImageTags != null && dto.ParentBackdropImageTags.Length > 0)))
@ -1305,7 +1369,7 @@ namespace Emby.Server.Implementations.Dto
if (images.Count > 0)
{
dto.ParentBackdropItemId = GetDtoId(parent);
dto.ParentBackdropImageTags = GetImageTags(parent, images);
dto.ParentBackdropImageTags = GetTagsAndFillBlurhashes(dto, parent, ImageType.Backdrop, images);
}
}

View File

@ -21,6 +21,7 @@ using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.Progress;
using MediaBrowser.Controller;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Drawing;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
@ -35,6 +36,7 @@ using MediaBrowser.Controller.Resolvers;
using MediaBrowser.Controller.Sorting;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Dlna;
using MediaBrowser.Model.Drawing;
using MediaBrowser.Model.Dto;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.IO;
@ -67,6 +69,7 @@ namespace Emby.Server.Implementations.Library
private readonly IFileSystem _fileSystem;
private readonly IItemRepository _itemRepository;
private readonly ConcurrentDictionary<Guid, BaseItem> _libraryItemsCache;
private readonly IImageProcessor _imageProcessor;
private NamingOptions _namingOptions;
private string[] _videoFileExtensions;
@ -147,7 +150,8 @@ namespace Emby.Server.Implementations.Library
Lazy<IProviderManager> providerManagerFactory,
Lazy<IUserViewManager> userviewManagerFactory,
IMediaEncoder mediaEncoder,
IItemRepository itemRepository)
IItemRepository itemRepository,
IImageProcessor imageProcessor)
{
_appHost = appHost;
_logger = logger;
@ -161,6 +165,7 @@ namespace Emby.Server.Implementations.Library
_userviewManagerFactory = userviewManagerFactory;
_mediaEncoder = mediaEncoder;
_itemRepository = itemRepository;
_imageProcessor = imageProcessor;
_libraryItemsCache = new ConcurrentDictionary<Guid, BaseItem>();
@ -1815,10 +1820,90 @@ namespace Emby.Server.Implementations.Library
}
}
public void UpdateImages(BaseItem item)
private bool ImageNeedsRefresh(ItemImageInfo image)
{
_itemRepository.SaveImages(item);
if (image.Path != null && image.IsLocalFile)
{
if (image.Width == 0 || image.Height == 0 || string.IsNullOrEmpty(image.BlurHash))
{
return true;
}
try
{
return _fileSystem.GetLastWriteTimeUtc(image.Path) != image.DateModified;
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot get file info for {0}", image.Path);
return false;
}
}
return image.Path != null && !image.IsLocalFile;
}
public void UpdateImages(BaseItem item, bool forceUpdate = false)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
var outdated = forceUpdate ? item.ImageInfos.Where(i => i.Path != null).ToArray() : item.ImageInfos.Where(ImageNeedsRefresh).ToArray();
if (outdated.Length == 0)
{
RegisterItem(item);
return;
}
foreach (var img in outdated)
{
var image = img;
if (!img.IsLocalFile)
{
try
{
var index = item.GetImageIndex(img);
image = ConvertImageToLocal(item, img, index).ConfigureAwait(false).GetAwaiter().GetResult();
}
catch (ArgumentException)
{
_logger.LogWarning("Cannot get image index for {0}", img.Path);
continue;
}
catch (InvalidOperationException)
{
_logger.LogWarning("Cannot fetch image from {0}", img.Path);
continue;
}
}
ImageDimensions size = _imageProcessor.GetImageDimensions(item, image);
image.Width = size.Width;
image.Height = size.Height;
try
{
image.BlurHash = _imageProcessor.GetImageBlurHash(image.Path);
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot compute blurhash for {0}", image.Path);
image.BlurHash = string.Empty;
}
try
{
image.DateModified = _fileSystem.GetLastWriteTimeUtc(image.Path);
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot update DateModified for {0}", image.Path);
}
}
_itemRepository.SaveImages(item);
RegisterItem(item);
}
@ -1839,7 +1924,7 @@ namespace Emby.Server.Implementations.Library
item.DateLastSaved = DateTime.UtcNow;
RegisterItem(item);
UpdateImages(item, updateReason >= ItemUpdateType.ImageUpdate);
}
_itemRepository.SaveItems(itemsList, cancellationToken);

View File

@ -18,6 +18,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="BlurHashSharp" Version="1.0.1" />
<PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.0.0" />
<PackageReference Include="SkiaSharp" Version="1.68.1" />
<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="1.68.1" />
<PackageReference Include="Jellyfin.SkiaSharp.NativeAssets.LinuxArm" Version="1.68.1" />

View File

@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using BlurHashSharp.SkiaSharp;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.Drawing;
using MediaBrowser.Controller.Extensions;
@ -229,6 +230,20 @@ namespace Jellyfin.Drawing.Skia
}
}
/// <inheritdoc />
/// <exception cref="ArgumentNullException">The path is null.</exception>
/// <exception cref="FileNotFoundException">The path is not valid.</exception>
/// <exception cref="SkiaCodecException">The file at the specified path could not be used to generate a codec.</exception>
public string GetImageBlurHash(int xComp, int yComp, string path)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
return BlurHashEncoder.Encode(xComp, yComp, path);
}
private static bool HasDiacritics(string text)
=> !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal);

View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Common.Extensions;
@ -280,9 +281,16 @@ namespace MediaBrowser.Api.Images
public List<ImageInfo> GetItemImageInfos(BaseItem item)
{
var list = new List<ImageInfo>();
var itemImages = item.ImageInfos;
if (itemImages.Length == 0)
{
// short-circuit
return list;
}
_libraryManager.UpdateImages(item); // this makes sure dimensions and hashes are correct
foreach (var image in itemImages)
{
if (!item.AllowsMultipleImages(image.Type))
@ -323,6 +331,7 @@ namespace MediaBrowser.Api.Images
{
int? width = null;
int? height = null;
string blurhash = null;
long length = 0;
try
@ -332,10 +341,9 @@ namespace MediaBrowser.Api.Images
var fileInfo = _fileSystem.GetFileInfo(info.Path);
length = fileInfo.Length;
ImageDimensions size = _imageProcessor.GetImageDimensions(item, info);
_libraryManager.UpdateImages(item);
width = size.Width;
height = size.Height;
blurhash = info.BlurHash;
width = info.Width;
height = info.Height;
if (width <= 0 || height <= 0)
{
@ -358,6 +366,7 @@ namespace MediaBrowser.Api.Images
ImageType = info.Type,
ImageTag = _imageProcessor.GetImageCacheTag(item, info),
Size = length,
BlurHash = blurhash,
Width = width,
Height = height
};

View File

@ -43,6 +43,15 @@ namespace MediaBrowser.Controller.Drawing
/// <returns>The image dimensions.</returns>
ImageDimensions GetImageSize(string path);
/// <summary>
/// Gets the blurhash of an image.
/// </summary>
/// <param name="xComp">Amount of X components of DCT to take.</param>
/// <param name="yComp">Amount of Y components of DCT to take.</param>
/// <param name="path">The filepath of the image.</param>
/// <returns>The blurhash.</returns>
string GetImageBlurHash(int xComp, int yComp, string path);
/// <summary>
/// Encode an image.
/// </summary>

View File

@ -40,6 +40,13 @@ namespace MediaBrowser.Controller.Drawing
/// <returns>ImageDimensions</returns>
ImageDimensions GetImageDimensions(BaseItem item, ItemImageInfo info);
/// <summary>
/// Gets the blurhash of the image.
/// </summary>
/// <param name="path">Path to the image file.</param>
/// <returns>BlurHash</returns>
string GetImageBlurHash(string path);
/// <summary>
/// Gets the image cache tag.
/// </summary>
@ -47,6 +54,7 @@ namespace MediaBrowser.Controller.Drawing
/// <param name="image">The image.</param>
/// <returns>Guid.</returns>
string GetImageCacheTag(BaseItem item, ItemImageInfo image);
string GetImageCacheTag(BaseItem item, ChapterInfo info);
/// <summary>

View File

@ -1374,6 +1374,7 @@ namespace MediaBrowser.Controller.Entities
new List<FileSystemMetadata>();
var ownedItemsChanged = await RefreshedOwnedItems(options, files, cancellationToken).ConfigureAwait(false);
LibraryManager.UpdateImages(this); // ensure all image properties in DB are fresh
if (ownedItemsChanged)
{
@ -2222,6 +2223,7 @@ namespace MediaBrowser.Controller.Entities
existingImage.DateModified = image.DateModified;
existingImage.Width = image.Width;
existingImage.Height = image.Height;
existingImage.BlurHash = image.BlurHash;
}
else
{
@ -2373,6 +2375,46 @@ namespace MediaBrowser.Controller.Entities
.ElementAtOrDefault(imageIndex);
}
/// <summary>
/// Computes image index for given image or raises if no matching image found.
/// </summary>
/// <param name="image">Image to compute index for.</param>
/// <exception cref="ArgumentException">Image index cannot be computed as no matching image found.
/// </exception>
/// <returns>Image index.</returns>
public int GetImageIndex(ItemImageInfo image)
{
if (image == null)
{
throw new ArgumentNullException(nameof(image));
}
if (image.Type == ImageType.Chapter)
{
var chapters = ItemRepository.GetChapters(this);
for (var i = 0; i < chapters.Count; i++)
{
if (chapters[i].ImagePath == image.Path)
{
return i;
}
}
throw new ArgumentException("No chapter index found for image path", image.Path);
}
var images = GetImages(image.Type).ToArray();
for (var i = 0; i < images.Length; i++)
{
if (images[i].Path == image.Path)
{
return i;
}
}
throw new ArgumentException("No image index found for image path", image.Path);
}
public IEnumerable<ItemImageInfo> GetImages(ImageType imageType)
{
if (imageType == ImageType.Chapter)

View File

@ -341,6 +341,11 @@ namespace MediaBrowser.Controller.Entities
{
currentChild.UpdateToRepository(ItemUpdateType.MetadataImport, cancellationToken);
}
else
{
// metadata is up-to-date; make sure DB has correct images dimensions and hash
LibraryManager.UpdateImages(currentChild);
}
continue;
}

View File

@ -28,6 +28,12 @@ namespace MediaBrowser.Controller.Entities
public int Height { get; set; }
/// <summary>
/// Gets or sets the blurhash.
/// </summary>
/// <value>The blurhash.</value>
public string BlurHash { get; set; }
[JsonIgnore]
public bool IsLocalFile => Path == null || !Path.StartsWith("http", StringComparison.OrdinalIgnoreCase);
}

View File

@ -118,7 +118,7 @@ namespace MediaBrowser.Controller.Library
/// </summary>
void QueueLibraryScan();
void UpdateImages(BaseItem item);
void UpdateImages(BaseItem item, bool forceUpdate = false);
/// <summary>
/// Gets the default view.
@ -195,6 +195,7 @@ namespace MediaBrowser.Controller.Library
/// Updates the item.
/// </summary>
void UpdateItems(IEnumerable<BaseItem> items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken);
void UpdateItem(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken);
/// <summary>

View File

@ -510,6 +510,13 @@ namespace MediaBrowser.Model.Dto
/// <value>The series thumb image tag.</value>
public string SeriesThumbImageTag { get; set; }
/// <summary>
/// Gets or sets the blurhashes for the image tags.
/// Maps image type to dictionary mapping image tag to blurhash value.
/// </summary>
/// <value>The blurhashes.</value>
public Dictionary<ImageType, Dictionary<string, string>> ImageBlurHashes { get; set; }
/// <summary>
/// Gets or sets the series studio.
/// </summary>

View File

@ -30,6 +30,12 @@ namespace MediaBrowser.Model.Dto
/// <value>The path.</value>
public string Path { get; set; }
/// <summary>
/// Gets or sets the blurhash.
/// </summary>
/// <value>The blurhash.</value>
public string BlurHash { get; set; }
/// <summary>
/// Gets or sets the height.
/// </summary>