From 763862cbd879aceed9277d79c5e38e851403cfe6 Mon Sep 17 00:00:00 2001 From: cvium Date: Sat, 3 Oct 2020 13:36:53 +0200 Subject: [PATCH 1/7] Defer image pre-fetching until the end of a refresh/scan --- .../Library/ImageFetcherPostScanTask.cs | 118 ++++++++++++++++++ .../Library/LibraryManager.cs | 45 +++++-- MediaBrowser.Controller/Entities/BaseItem.cs | 1 - MediaBrowser.Controller/Entities/Folder.cs | 5 - .../Library/ILibraryManager.cs | 2 + .../Manager/MetadataService.cs | 10 +- 6 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs new file mode 100644 index 000000000..94d7f3cd6 --- /dev/null +++ b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Concurrent; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Events; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; +using Microsoft.Extensions.Logging; + +namespace Emby.Server.Implementations.Library +{ + /// + /// Test. + /// + public class ImageFetcherPostScanTask : ILibraryPostScanTask + { + private readonly ILibraryManager _libraryManager; + private readonly IProviderManager _providerManager; + private readonly ILogger _logger; + private readonly SemaphoreSlim _imageFetcherLock; + + private ConcurrentDictionary _queuedItems; + + /// + /// Initializes a new instance of the class. + /// + /// Some stuff. + public ImageFetcherPostScanTask( + ILibraryManager libraryManager, + IProviderManager providerManager, + ILogger logger) + { + _libraryManager = libraryManager; + _providerManager = providerManager; + _logger = logger; + _queuedItems = new ConcurrentDictionary(); + _imageFetcherLock = new SemaphoreSlim(1, 1); + _libraryManager.ItemAdded += OnLibraryManagerItemAddedOrUpdated; + _libraryManager.ItemUpdated += OnLibraryManagerItemAddedOrUpdated; + _providerManager.RefreshCompleted += OnProviderManagerRefreshCompleted; + } + + /// + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + // Sometimes a library scan will cause this to run twice if there's an item refresh going on. + await _imageFetcherLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + var now = DateTime.UtcNow; + var itemGuids = _queuedItems.Keys.ToList(); + + for (var i = 0; i < itemGuids.Count; i++) + { + if (!_queuedItems.TryGetValue(itemGuids[i], out var queuedItem)) + { + continue; + } + + _logger.LogDebug( + "Updating remote images for item {ItemId} with media type {ItemMediaType}", + queuedItem.item.Id.ToString("N", CultureInfo.InvariantCulture), + queuedItem.item.GetType()); + await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); + + _queuedItems.TryRemove(queuedItem.item.Id, out _); + } + + if (itemGuids.Count > 0) + { + _logger.LogInformation( + "Finished updating/pre-fetching {NumberOfImages} images. Elapsed time: {TimeElapsed}s.", + itemGuids.Count.ToString(CultureInfo.InvariantCulture), + (DateTime.UtcNow - now).TotalSeconds.ToString(CultureInfo.InvariantCulture)); + } + else + { + _logger.LogDebug("No images were updated."); + } + } + finally + { + _imageFetcherLock.Release(); + } + } + + private void OnLibraryManagerItemAddedOrUpdated(object sender, ItemChangeEventArgs itemChangeEventArgs) + { + if (!_queuedItems.ContainsKey(itemChangeEventArgs.Item.Id) && itemChangeEventArgs.Item.ImageInfos.Length > 0) + { + _queuedItems.AddOrUpdate( + itemChangeEventArgs.Item.Id, + (itemChangeEventArgs.Item, itemChangeEventArgs.UpdateReason), + (key, existingValue) => existingValue); + } + } + + private void OnProviderManagerRefreshCompleted(object sender, GenericEventArgs e) + { + if (!_queuedItems.ContainsKey(e.Argument.Id) && e.Argument.ImageInfos.Length > 0) + { + _queuedItems.AddOrUpdate( + e.Argument.Id, + (e.Argument, ItemUpdateType.None), + (key, existingValue) => existingValue); + } + + // The RefreshCompleted event is a bit awkward in that it seems to _only_ be fired on + // the item that was refreshed regardless of children refreshes. So we take it as a signal + // that the refresh is entirely completed. + Run(null, CancellationToken.None).GetAwaiter().GetResult(); + } + } +} diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 00282b71a..74788a320 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -857,7 +857,21 @@ namespace Emby.Server.Implementations.Library /// Task{Person}. public Person GetPerson(string name) { - return CreateItemByName(Person.GetPath, name, new DtoOptions(true)); + var path = Person.GetPath(name); + var id = GetItemByNameId(path); + if (!(GetItemById(id) is Person item)) + { + item = new Person + { + Name = name, + Id = id, + DateCreated = DateTime.UtcNow, + DateModified = DateTime.UtcNow, + Path = path + }; + } + + return item; } /// @@ -1940,19 +1954,9 @@ namespace Emby.Server.Implementations.Library } /// - public async Task UpdateItemsAsync(IReadOnlyList items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) + public Task UpdateItemsAsync(IReadOnlyList items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) { - foreach (var item in items) - { - if (item.IsFileProtocol) - { - ProviderManager.SaveMetadata(item, updateReason); - } - - item.DateLastSaved = DateTime.UtcNow; - - await UpdateImagesAsync(item, updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); - } + RunMetadataSavers(items, updateReason); _itemRepository.SaveItems(items, cancellationToken); @@ -1983,12 +1987,27 @@ namespace Emby.Server.Implementations.Library } } } + + return Task.CompletedTask; } /// public Task UpdateItemAsync(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) => UpdateItemsAsync(new[] { item }, parent, updateReason, cancellationToken); + public void RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason) + { + foreach (var item in items) + { + if (item.IsFileProtocol) + { + ProviderManager.SaveMetadata(item, updateReason); + } + + item.DateLastSaved = DateTime.UtcNow; + } + } + /// /// Reports the item removed. /// diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 68126bd8a..0e9964608 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -1433,7 +1433,6 @@ namespace MediaBrowser.Controller.Entities new List(); var ownedItemsChanged = await RefreshedOwnedItems(options, files, cancellationToken).ConfigureAwait(false); - await LibraryManager.UpdateImagesAsync(this).ConfigureAwait(false); // ensure all image properties in DB are fresh if (ownedItemsChanged) { diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 11542c1ca..b8737b48c 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -352,11 +352,6 @@ namespace MediaBrowser.Controller.Entities { await currentChild.UpdateToRepositoryAsync(ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false); } - else - { - // metadata is up-to-date; make sure DB has correct images dimensions and hash - await LibraryManager.UpdateImagesAsync(currentChild).ConfigureAwait(false); - } continue; } diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 804170d5c..d329495b9 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -567,5 +567,7 @@ namespace MediaBrowser.Controller.Library void AddExternalSubtitleStreams(List streams, string videoPath, string[] files); + + void RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason); } } diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index f110eafa5..7ca51f4b5 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -231,14 +231,14 @@ namespace MediaBrowser.Providers.Manager private async Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) { + var personsToSave = new List(); + foreach (var person in people) { cancellationToken.ThrowIfCancellationRequested(); if (person.ProviderIds.Count > 0 || !string.IsNullOrWhiteSpace(person.ImageUrl)) { - var updateType = ItemUpdateType.MetadataDownload; - var saveEntity = false; var personEntity = LibraryManager.GetPerson(person.Name); foreach (var id in person.ProviderIds) @@ -255,15 +255,17 @@ namespace MediaBrowser.Providers.Manager await AddPersonImageAsync(personEntity, libraryOptions, person.ImageUrl, cancellationToken).ConfigureAwait(false); saveEntity = true; - updateType |= ItemUpdateType.ImageUpdate; } if (saveEntity) { - await personEntity.UpdateToRepositoryAsync(updateType, cancellationToken).ConfigureAwait(false); + personsToSave.Add(personEntity); } } } + + LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload); + LibraryManager.CreateItems(personsToSave, null, CancellationToken.None); } private async Task AddPersonImageAsync(Person personEntity, LibraryOptions libraryOptions, string imageUrl, CancellationToken cancellationToken) From 5d3449e9dc8c43efa16669a390870c52676bcba5 Mon Sep 17 00:00:00 2001 From: cvium Date: Sat, 3 Oct 2020 13:39:39 +0200 Subject: [PATCH 2/7] Fix xml doc comment --- Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs index 94d7f3cd6..49f920eda 100644 --- a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs +++ b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs @@ -13,7 +13,7 @@ using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library { /// - /// Test. + /// A library post scan/refresh task for pre-fetching remote images. /// public class ImageFetcherPostScanTask : ILibraryPostScanTask { From 1b18f86c8b92eb0c75ee6db27eaffce918afa988 Mon Sep 17 00:00:00 2001 From: cvium Date: Sat, 3 Oct 2020 13:52:51 +0200 Subject: [PATCH 3/7] Add missing parameter comments. --- .../Library/ImageFetcherPostScanTask.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs index 49f920eda..66540b4d4 100644 --- a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs +++ b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs @@ -27,7 +27,9 @@ namespace Emby.Server.Implementations.Library /// /// Initializes a new instance of the class. /// - /// Some stuff. + /// An instance of . + /// An instance of . + /// An instance of . public ImageFetcherPostScanTask( ILibraryManager libraryManager, IProviderManager providerManager, From 4a81ee43dc7aed94012c312a8262a1426be9b6d9 Mon Sep 17 00:00:00 2001 From: cvium Date: Tue, 6 Oct 2020 23:36:48 +0200 Subject: [PATCH 4/7] Add try-catch to avoid crashing the whole thing --- .../Library/ImageFetcherPostScanTask.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs index 66540b4d4..b18a0c1a8 100644 --- a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs +++ b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs @@ -8,6 +8,7 @@ using Jellyfin.Data.Events; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Net; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library @@ -63,11 +64,20 @@ namespace Emby.Server.Implementations.Library continue; } + var itemId = queuedItem.item.Id.ToString("N", CultureInfo.InvariantCulture); + var itemType = queuedItem.item.GetType(); _logger.LogDebug( "Updating remote images for item {ItemId} with media type {ItemMediaType}", - queuedItem.item.Id.ToString("N", CultureInfo.InvariantCulture), - queuedItem.item.GetType()); - await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); + itemId, + itemType); + try + { + await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); + } + catch (HttpException ex) + { + _logger.LogError(ex, "Failed to fetch images for {Type} item with id {ItemId}", itemType, itemId); + } _queuedItems.TryRemove(queuedItem.item.Id, out _); } From 79855a76b1eca5c6aee3afc596f1569167862dd0 Mon Sep 17 00:00:00 2001 From: BaronGreenback Date: Sat, 21 Nov 2020 23:21:15 +0000 Subject: [PATCH 5/7] Update MetadataService.cs removed async line 232 --- MediaBrowser.Providers/Manager/MetadataService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 3542f90e1..ba5930323 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -229,7 +229,7 @@ namespace MediaBrowser.Providers.Manager await result.Item.UpdateToRepositoryAsync(reason, cancellationToken).ConfigureAwait(false); } - private async Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) + private Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) { var personsToSave = new List(); From d3b0080a27152337d26e2d94d8f3421d314fbf8b Mon Sep 17 00:00:00 2001 From: BaronGreenback Date: Sat, 21 Nov 2020 23:29:52 +0000 Subject: [PATCH 6/7] Update MetadataService.cs added return Task.CompletedTask; --- MediaBrowser.Providers/Manager/MetadataService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index ba5930323..6dbce3067 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -272,6 +272,7 @@ namespace MediaBrowser.Providers.Manager LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload); LibraryManager.CreateItems(personsToSave, null, CancellationToken.None); + return Task.CompletedTask; } protected virtual Task AfterMetadataRefresh(TItemType item, MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken) From 969b9e2a18aa76fd0b2915960ea799a12a9605a0 Mon Sep 17 00:00:00 2001 From: BaronGreenback Date: Sat, 21 Nov 2020 23:33:41 +0000 Subject: [PATCH 7/7] Update ImageFetcherPostScanTask.cs --- Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs index b18a0c1a8..d4e790c9a 100644 --- a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs +++ b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs @@ -74,7 +74,7 @@ namespace Emby.Server.Implementations.Library { await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); } - catch (HttpException ex) + catch (Exception ex) { _logger.LogError(ex, "Failed to fetch images for {Type} item with id {ItemId}", itemType, itemId); }