Merge pull request #4263 from cvium/defer_image_fetching

Defer image pre-fetching until the end of a refresh/scan

Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Concurrent;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Jellyfin.Data.Events;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model.Net;
+using Microsoft.Extensions.Logging;
+
+namespace Emby.Server.Implementations.Library
+{
+    /// <summary>
+    /// A library post scan/refresh task for pre-fetching remote images.
+    /// </summary>
+    public class ImageFetcherPostScanTask : ILibraryPostScanTask
+    {
+        private readonly ILibraryManager _libraryManager;
+        private readonly IProviderManager _providerManager;
+        private readonly ILogger<ImageFetcherPostScanTask> _logger;
+        private readonly SemaphoreSlim _imageFetcherLock;
+
+        private ConcurrentDictionary<Guid, (BaseItem item, ItemUpdateType updateReason)> _queuedItems;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ImageFetcherPostScanTask"/> class.
+        /// </summary>
+        /// <param name="libraryManager">An instance of <see cref="ILibraryManager"/>.</param>
+        /// <param name="providerManager">An instance of <see cref="IProviderManager"/>.</param>
+        /// <param name="logger">An instance of <see cref="ILogger{ImageFetcherPostScanTask}"/>.</param>
+        public ImageFetcherPostScanTask(
+            ILibraryManager libraryManager,
+            IProviderManager providerManager,
+            ILogger<ImageFetcherPostScanTask> logger)
+        {
+            _libraryManager = libraryManager;
+            _providerManager = providerManager;
+            _logger = logger;
+            _queuedItems = new ConcurrentDictionary<Guid, (BaseItem item, ItemUpdateType updateReason)>();
+            _imageFetcherLock = new SemaphoreSlim(1, 1);
+            _libraryManager.ItemAdded += OnLibraryManagerItemAddedOrUpdated;
+            _libraryManager.ItemUpdated += OnLibraryManagerItemAddedOrUpdated;
+            _providerManager.RefreshCompleted += OnProviderManagerRefreshCompleted;
+        }
+
+        /// <inheritdoc />
+        public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            // Sometimes a library scan will cause this to run twice if there's an item refresh going on.
+            await _imageFetcherLock.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            try
+            {
+                var now = DateTime.UtcNow;
+                var itemGuids = _queuedItems.Keys.ToList();
+
+                for (var i = 0; i < itemGuids.Count; i++)
+                {
+                    if (!_queuedItems.TryGetValue(itemGuids[i], out var queuedItem))
+                    {
+                        continue;
+                    }
+
+                    var itemId = queuedItem.item.Id.ToString("N", CultureInfo.InvariantCulture);
+                    var itemType = queuedItem.item.GetType();
+                    _logger.LogDebug(
+                        "Updating remote images for item {ItemId} with media type {ItemMediaType}",
+                        itemId,
+                        itemType);
+                    try
+                    {
+                        await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "Failed to fetch images for {Type} item with id {ItemId}", itemType, itemId);
+                    }
+
+                    _queuedItems.TryRemove(queuedItem.item.Id, out _);
+                }
+
+                if (itemGuids.Count > 0)
+                {
+                    _logger.LogInformation(
+                        "Finished updating/pre-fetching {NumberOfImages} images. Elapsed time: {TimeElapsed}s.",
+                        itemGuids.Count.ToString(CultureInfo.InvariantCulture),
+                        (DateTime.UtcNow - now).TotalSeconds.ToString(CultureInfo.InvariantCulture));
+                }
+                else
+                {
+                    _logger.LogDebug("No images were updated.");
+                }
+            }
+            finally
+            {
+                _imageFetcherLock.Release();
+            }
+        }
+
+        private void OnLibraryManagerItemAddedOrUpdated(object sender, ItemChangeEventArgs itemChangeEventArgs)
+        {
+            if (!_queuedItems.ContainsKey(itemChangeEventArgs.Item.Id) && itemChangeEventArgs.Item.ImageInfos.Length > 0)
+            {
+                _queuedItems.AddOrUpdate(
+                    itemChangeEventArgs.Item.Id,
+                    (itemChangeEventArgs.Item, itemChangeEventArgs.UpdateReason),
+                    (key, existingValue) => existingValue);
+            }
+        }
+
+        private void OnProviderManagerRefreshCompleted(object sender, GenericEventArgs<BaseItem> e)
+        {
+            if (!_queuedItems.ContainsKey(e.Argument.Id) && e.Argument.ImageInfos.Length > 0)
+            {
+                _queuedItems.AddOrUpdate(
+                    e.Argument.Id,
+                    (e.Argument, ItemUpdateType.None),
+                    (key, existingValue) => existingValue);
+            }
+
+            // The RefreshCompleted event is a bit awkward in that it seems to _only_ be fired on
+            // the item that was refreshed regardless of children refreshes. So we take it as a signal
+            // that the refresh is entirely completed.
+            Run(null, CancellationToken.None).GetAwaiter().GetResult();
+        }
+    }
+}

Emby.Server.Implementations/Library/LibraryManager.cs

@@ -858,7 +858,21 @@ namespace Emby.Server.Implementations.Library
         /// <returns>Task{Person}.</returns>
         public Person GetPerson(string name)
         {
-            return CreateItemByName<Person>(Person.GetPath, name, new DtoOptions(true));
+            var path = Person.GetPath(name);
+            var id = GetItemByNameId<Person>(path);
+            if (!(GetItemById(id) is Person item))
+            {
+                item = new Person
+                {
+                    Name = name,
+                    Id = id,
+                    DateCreated = DateTime.UtcNow,
+                    DateModified = DateTime.UtcNow,
+                    Path = path
+                };
+            }
+
+            return item;
         }
 
         /// <summary>
@@ -1941,19 +1955,9 @@ namespace Emby.Server.Implementations.Library
         }
 
         /// <inheritdoc />
-        public async Task UpdateItemsAsync(IReadOnlyList<BaseItem> items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken)
+        public Task UpdateItemsAsync(IReadOnlyList<BaseItem> items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken)
         {
-            foreach (var item in items)
-            {
-                if (item.IsFileProtocol)
-                {
-                    ProviderManager.SaveMetadata(item, updateReason);
-                }
-
-                item.DateLastSaved = DateTime.UtcNow;
-
-                await UpdateImagesAsync(item, updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false);
-            }
+            RunMetadataSavers(items, updateReason);
 
             _itemRepository.SaveItems(items, cancellationToken);
 
@@ -1984,12 +1988,27 @@ namespace Emby.Server.Implementations.Library
                     }
                 }
             }
+
+            return Task.CompletedTask;
         }
 
         /// <inheritdoc />
         public Task UpdateItemAsync(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken)
             => UpdateItemsAsync(new[] { item }, parent, updateReason, cancellationToken);
 
+        public void RunMetadataSavers(IReadOnlyList<BaseItem> items, ItemUpdateType updateReason)
+        {
+            foreach (var item in items)
+            {
+                if (item.IsFileProtocol)
+                {
+                    ProviderManager.SaveMetadata(item, updateReason);
+                }
+
+                item.DateLastSaved = DateTime.UtcNow;
+            }
+        }
+
         /// <summary>
         /// Reports the item removed.
         /// </summary>

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -1385,7 +1385,6 @@ namespace MediaBrowser.Controller.Entities
                         new List<FileSystemMetadata>();
 
                     var ownedItemsChanged = await RefreshedOwnedItems(options, files, cancellationToken).ConfigureAwait(false);
-                    await LibraryManager.UpdateImagesAsync(this).ConfigureAwait(false); // ensure all image properties in DB are fresh
 
                     if (ownedItemsChanged)
                     {

MediaBrowser.Controller/Entities/Folder.cs

@@ -353,11 +353,6 @@ namespace MediaBrowser.Controller.Entities
                         {
                             await currentChild.UpdateToRepositoryAsync(ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false);
                         }
-                        else
-                        {
-                            // metadata is up-to-date; make sure DB has correct images dimensions and hash
-                            await LibraryManager.UpdateImagesAsync(currentChild).ConfigureAwait(false);
-                        }
 
                         continue;
                     }

MediaBrowser.Controller/Library/ILibraryManager.cs

@@ -571,6 +571,8 @@ namespace MediaBrowser.Controller.Library
             string videoPath,
             string[] files);
 
+        void RunMetadataSavers(IReadOnlyList<BaseItem> items, ItemUpdateType updateReason);
+
         BaseItem GetParentItem(string parentId, Guid? userId);
     }
 }

MediaBrowser.Providers/Manager/MetadataService.cs

@@ -229,16 +229,16 @@ namespace MediaBrowser.Providers.Manager
             await result.Item.UpdateToRepositoryAsync(reason, cancellationToken).ConfigureAwait(false);
         }
 
-        private async Task SavePeopleMetadataAsync(List<PersonInfo> people, LibraryOptions libraryOptions, CancellationToken cancellationToken)
+        private Task SavePeopleMetadataAsync(List<PersonInfo> people, LibraryOptions libraryOptions, CancellationToken cancellationToken)
         {
+            var personsToSave = new List<BaseItem>();
+
             foreach (var person in people)
             {
                 cancellationToken.ThrowIfCancellationRequested();
 
                 if (person.ProviderIds.Count > 0 || !string.IsNullOrWhiteSpace(person.ImageUrl))
                 {
-                    var updateType = ItemUpdateType.MetadataDownload;
-
                     var saveEntity = false;
                     var personEntity = LibraryManager.GetPerson(person.Name);
                     foreach (var id in person.ProviderIds)
@@ -261,15 +261,18 @@ namespace MediaBrowser.Providers.Manager
                             0);
 
                         saveEntity = true;
-                        updateType |= ItemUpdateType.ImageUpdate;
                     }
 
                     if (saveEntity)
                     {
-                        await personEntity.UpdateToRepositoryAsync(updateType, cancellationToken).ConfigureAwait(false);
+                        personsToSave.Add(personEntity);
                     }
                 }
             }
+
+            LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload);
+            LibraryManager.CreateItems(personsToSave, null, CancellationToken.None);
+            return Task.CompletedTask;
         }
 
         protected virtual Task AfterMetadataRefresh(TItemType item, MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken)