Auto-Organize: Async operation and instant feedback UI (reworked)

This commit includes changes to enable and stabilize asyncronous
operation in the auto-organize area. Here are the key points:

- The auto-organize correction dialog is now closed (almost) instantly.
This means that the user does not have to wait until the file copy/move
operation is completed in order to continue. (even with local HDs the
copy/move process can take several minutes or even much longer with
network destination).
- This commit also implements locking of files to be organized in order
to prevent parallel processing of the same item. In effect, there can be
2 or more manual organization operations active even while the normal
auto-organization task is running without causing any problems
- The items that are currently being processed are indicated as such in
the log with an orange color and a spinner graphic
- The client display is refreshed through websocket messages
- A side effect of this is that other clients showing the auto-organize
log at the same time are always up-to-date as well

MediaBrowser.Api/Library/FileOrganizationService.cs

@@ -154,9 +154,12 @@ namespace MediaBrowser.Api.Library
 
         public void Post(PerformOrganization request)
         {
+            // Don't await this
             var task = _iFileOrganizationService.PerformOrganization(request.Id);
 
-            Task.WaitAll(task);
+            // Async processing (close dialog early instead of waiting until the file has been copied)
+            // Wait 2s for exceptions that may occur to have them forwarded to the client for immediate error display
+            task.Wait(2000);
         }
 
         public void Post(OrganizeEpisode request)
@@ -168,6 +171,7 @@ namespace MediaBrowser.Api.Library
                 dicNewProviderIds = request.NewSeriesProviderIds;
             }
 
+            // Don't await this
             var task = _iFileOrganizationService.PerformEpisodeOrganization(new EpisodeFileOrganizationRequest
             {
                 EndingEpisodeNumber = request.EndingEpisodeNumber,
@@ -182,11 +186,9 @@ namespace MediaBrowser.Api.Library
                 TargetFolder = request.TargetFolder
             });
 
-            // For async processing (close dialog early instead of waiting until the file has been copied)
+            // Async processing (close dialog early instead of waiting until the file has been copied)
-            //var tasks = new Task[] { task };
+            // Wait 2s for exceptions that may occur to have them forwarded to the client for immediate error display
-            //Task.WaitAll(tasks, 8000);
+            task.Wait(2000);
-
-            Task.WaitAll(task);
         }
 
         public object Get(GetSmartMatchInfos request)

MediaBrowser.Controller/FileOrganization/IFileOrganizationService.cs

@@ -1,5 +1,7 @@
-using MediaBrowser.Model.FileOrganization;
+using MediaBrowser.Model.Events;
+using MediaBrowser.Model.FileOrganization;
 using MediaBrowser.Model.Querying;
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -7,6 +9,11 @@ namespace MediaBrowser.Controller.FileOrganization
 {
     public interface IFileOrganizationService
     {
+        event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemAdded;
+        event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemUpdated;
+        event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemRemoved;
+        event EventHandler LogReset;
+
         /// <summary>
         /// Processes the new files.
         /// </summary>
@@ -81,5 +88,20 @@ namespace MediaBrowser.Controller.FileOrganization
         /// <param name="ItemName">Item name.</param>
         /// <param name="matchString">The match string to delete.</param>
         void DeleteSmartMatchEntry(string ItemName, string matchString);
+
+        /// <summary>
+        /// Attempts to add a an item to the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <param name="fullClientRefresh">Passing true will notify the client to reload all items, otherwise only a single item will be refreshed.</param>
+        /// <returns>True if the item was added, False if the item is already contained in the list.</returns>
+        bool AddToInProgressList(FileOrganizationResult result, bool fullClientRefresh);
+
+        /// <summary>
+        /// Removes an item from the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <returns>True if the item was removed, False if the item was not contained in the list.</returns>
+        bool RemoveFromInprogressList(FileOrganizationResult result);
     }
 }

MediaBrowser.Controller/Session/ISessionManager.cs

@@ -171,6 +171,16 @@ namespace MediaBrowser.Controller.Session
         /// <returns>Task.</returns>
         Task SendPlaystateCommand(string controllingSessionId, string sessionId, PlaystateRequest command, CancellationToken cancellationToken);
 
+        /// <summary>
+        /// Sends the message to admin sessions.
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="name">The name.</param>
+        /// <param name="data">The data.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        Task SendMessageToAdminSessions<T>(string name, T data, CancellationToken cancellationToken);
+
         /// <summary>
         /// Sends the message to user sessions.
         /// </summary>

MediaBrowser.Model/FileOrganization/FileOrganizationResult.cs

@@ -95,6 +95,12 @@ namespace MediaBrowser.Model.FileOrganization
         /// <value>The size of the file.</value>
         public long FileSize { get; set; }
 
+        /// <summary>
+        /// Indicates if the item is currently being processed.
+        /// </summary>
+        /// <remarks>Runtime property not persisted to the store.</remarks>
+        public bool IsInProgress { get; set; }
+
         public FileOrganizationResult()
         {
             DuplicatePaths = new List<string>();

MediaBrowser.Server.Implementations/FileOrganization/EpisodeFileOrganizer.cs

@@ -272,6 +272,18 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
 
             var originalExtractedSeriesString = result.ExtractedName;
 
+            bool isNew = string.IsNullOrWhiteSpace(result.Id);
+
+            if (isNew)
+            {
+                await _organizationService.SaveResult(result, cancellationToken);
+            }
+
+            if (!_organizationService.AddToInProgressList(result, isNew))
+            {
+                throw new Exception("File is currently processed otherwise. Please try again later.");
+            }
+            
             try
             {
             // Proceed to sort the file
@@ -363,6 +375,10 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
                 _logger.Warn(ex.Message);
                 return;
             }
+            finally
+            {
+                _organizationService.RemoveFromInprogressList(result);
+            }
 
             if (rememberCorrection)
             {

MediaBrowser.Server.Implementations/FileOrganization/FileOrganizationNotifier.cs

@@ -0,0 +1,68 @@
+using MediaBrowser.Controller.FileOrganization;
+using MediaBrowser.Controller.Plugins;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MediaBrowser.Model.Events;
+using MediaBrowser.Model.FileOrganization;
+using MediaBrowser.Controller.Session;
+using System.Threading;
+
+namespace MediaBrowser.Server.Implementations.FileOrganization
+{
+    /// <summary>
+    /// Class SessionInfoWebSocketListener
+    /// </summary>
+    class FileOrganizationNotifier : IServerEntryPoint
+    {
+        private readonly IFileOrganizationService _organizationService;
+        private readonly ISessionManager _sessionManager;
+
+        public FileOrganizationNotifier(ILogger logger, IFileOrganizationService organizationService, ISessionManager sessionManager)
+        {
+            _organizationService = organizationService;
+            _sessionManager = sessionManager;
+        }
+
+        public void Run()
+        {
+            _organizationService.ItemAdded += _organizationService_ItemAdded;
+            _organizationService.ItemRemoved += _organizationService_ItemRemoved;
+            _organizationService.ItemUpdated += _organizationService_ItemUpdated;
+            _organizationService.LogReset += _organizationService_LogReset;
+        }
+
+        private void _organizationService_LogReset(object sender, EventArgs e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemUpdated(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", e.Argument, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemRemoved(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemAdded(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        public void Dispose()
+        {
+            _organizationService.ItemAdded -= _organizationService_ItemAdded;
+            _organizationService.ItemRemoved -= _organizationService_ItemRemoved;
+            _organizationService.ItemUpdated -= _organizationService_ItemUpdated;
+            _organizationService.LogReset -= _organizationService_LogReset;
+        }
+
+
+    }
+}

MediaBrowser.Server.Implementations/FileOrganization/FileOrganizationService.cs

@@ -3,16 +3,21 @@ using MediaBrowser.Common.ScheduledTasks;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.FileOrganization;
 using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Net;
 using MediaBrowser.Controller.Persistence;
 using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.FileOrganization;
 using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.Querying;
 using System;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using CommonIO;
+using MediaBrowser.Controller.Session;
+using MediaBrowser.Model.Events;
+using MediaBrowser.Common.Events;
 
 namespace MediaBrowser.Server.Implementations.FileOrganization
 {
@@ -26,6 +31,12 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
         private readonly IServerConfigurationManager _config;
         private readonly IFileSystem _fileSystem;
         private readonly IProviderManager _providerManager;
+        private readonly ConcurrentDictionary<string, bool> _inProgressItemIds = new ConcurrentDictionary<string, bool>();
+
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemAdded;
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemUpdated;
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemRemoved;
+        public event EventHandler LogReset;
 
         public FileOrganizationService(ITaskManager taskManager, IFileOrganizationRepository repo, ILogger logger, ILibraryMonitor libraryMonitor, ILibraryManager libraryManager, IServerConfigurationManager config, IFileSystem fileSystem, IProviderManager providerManager)
         {
@@ -58,12 +69,26 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
 
         public QueryResult<FileOrganizationResult> GetResults(FileOrganizationResultQuery query)
         {
-            return _repo.GetResults(query);
+            var results = _repo.GetResults(query);
+
+            foreach (var result in results.Items)
+            {
+                result.IsInProgress = _inProgressItemIds.ContainsKey(result.Id);
+            }
+
+            return results;
         }
 
         public FileOrganizationResult GetResult(string id)
         {
-            return _repo.GetResult(id);
+            var result = _repo.GetResult(id);
+
+            if (result != null)
+            {
+                result.IsInProgress = _inProgressItemIds.ContainsKey(result.Id);
+            }
+
+            return result;
         }
 
         public FileOrganizationResult GetResultBySourcePath(string path)
@@ -78,11 +103,17 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             return GetResult(id);
         }
 
-        public Task DeleteOriginalFile(string resultId)
+        public async Task DeleteOriginalFile(string resultId)
         {
             var result = _repo.GetResult(resultId);
 
             _logger.Info("Requested to delete {0}", result.OriginalPath);
+
+            if (!AddToInProgressList(result, false))
+            {
+                throw new Exception("Path is currently processed otherwise. Please try again later.");
+            }
+
             try
             {
                 _fileSystem.DeleteFile(result.OriginalPath);
@@ -91,8 +122,14 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             {
                 _logger.ErrorException("Error deleting {0}", ex, result.OriginalPath);
             }
+            finally
+            {
+                RemoveFromInprogressList(result);
+            }
 
-            return _repo.Delete(resultId);
+            await _repo.Delete(resultId);
+
+            EventHelper.FireEventIfNotNull(ItemRemoved, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
         }
 
         private AutoOrganizeOptions GetAutoOrganizeOptions()
@@ -121,9 +158,10 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             }
         }
 
-        public Task ClearLog()
+        public async Task ClearLog()
         {
-            return _repo.DeleteAll();
+            await _repo.DeleteAll();
+            EventHelper.FireEventIfNotNull(LogReset, this, EventArgs.Empty, _logger);
         }
 
         public async Task PerformEpisodeOrganization(EpisodeFileOrganizationRequest request)
@@ -189,5 +227,55 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
                 _config.SaveAutoOrganizeOptions(options);
             }
         }
+
+        /// <summary>
+        /// Attempts to add a an item to the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <param name="isNewItem">Passing true will notify the client to reload all items, otherwise only a single item will be refreshed.</param>
+        /// <returns>True if the item was added, False if the item is already contained in the list.</returns>
+        public bool AddToInProgressList(FileOrganizationResult result, bool isNewItem)
+        {
+            if (string.IsNullOrWhiteSpace(result.Id))
+            {
+                result.Id = result.OriginalPath.GetMD5().ToString("N");
+            }
+
+            if (!_inProgressItemIds.TryAdd(result.Id, false))
+            {
+                return false;
+            }
+
+            result.IsInProgress = true;
+
+            if (isNewItem)
+            {
+                EventHelper.FireEventIfNotNull(ItemAdded, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+            }
+            else
+            {
+                EventHelper.FireEventIfNotNull(ItemUpdated, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Removes an item from the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <returns>True if the item was removed, False if the item was not contained in the list.</returns>
+        public bool RemoveFromInprogressList(FileOrganizationResult result)
+        {
+            bool itemValue;
+            var retval = _inProgressItemIds.TryRemove(result.Id, out itemValue);
+
+            result.IsInProgress = false;
+
+            EventHelper.FireEventIfNotNull(ItemUpdated, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+
+            return retval;
+        }
+
     }
 }

MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj

@@ -149,6 +149,7 @@
     <Compile Include="EntryPoints\UsageReporter.cs" />
     <Compile Include="FileOrganization\EpisodeFileOrganizer.cs" />
     <Compile Include="FileOrganization\Extensions.cs" />
+    <Compile Include="FileOrganization\FileOrganizationNotifier.cs" />
     <Compile Include="FileOrganization\FileOrganizationService.cs" />
     <Compile Include="FileOrganization\NameUtils.cs" />
     <Compile Include="FileOrganization\TvFolderOrganizer.cs" />

MediaBrowser.Server.Implementations/Session/SessionManager.cs

@@ -1869,6 +1869,27 @@ namespace MediaBrowser.Server.Implementations.Session
             return GetSessionByAuthenticationToken(info, deviceId, remoteEndpoint, null);
         }
 
+        public Task SendMessageToAdminSessions<T>(string name, T data, CancellationToken cancellationToken)
+        {
+            // TODO: How to identify admin sessions?
+            var sessions = Sessions.Where(i => i.IsActive && i.SessionController != null).ToList();
+
+            var tasks = sessions.Select(session => Task.Run(async () =>
+            {
+                try
+                {
+                    await session.SessionController.SendMessage(name, data, cancellationToken).ConfigureAwait(false);
+                }
+                catch (Exception ex)
+                {
+                    _logger.ErrorException("Error sending message", ex);
+                }
+
+            }, cancellationToken));
+
+            return Task.WhenAll(tasks);
+        }
+
         public Task SendMessageToUserSessions<T>(string userId, string name, T data,
             CancellationToken cancellationToken)
         {

MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj

@@ -182,6 +182,9 @@
     <Content Include="dashboard-ui\components\tvproviders\xmltv.template.html">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="dashboard-ui\css\images\throbber.gif">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="dashboard-ui\scripts\userpasswordpage.js">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

MediaBrowser.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.24720.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{F0E0E64C-2A6F-4E35-9533-D53AC07C2CD1}"
 EndProject
@@ -65,9 +65,6 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Emby.Drawing", "Emby.Drawing\Emby.Drawing.csproj", "{08FFF49B-F175-4807-A2B5-73B0EBD9F716}"
 EndProject
 Global
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Debug|Mixed Platforms = Debug|Mixed Platforms