From 6b31083e435b24e7a7accc4ba190940457e18477 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Thu, 26 May 2016 16:54:05 -0400 Subject: [PATCH] separate file monitor to avoid paths getting stuck --- .../IO/FileRefresher.cs | 49 ++- .../IO/LibraryMonitor.cs | 284 ++++-------------- 2 files changed, 108 insertions(+), 225 deletions(-) diff --git a/MediaBrowser.Server.Implementations/IO/FileRefresher.cs b/MediaBrowser.Server.Implementations/IO/FileRefresher.cs index 74dfbc679..18c52ab29 100644 --- a/MediaBrowser.Server.Implementations/IO/FileRefresher.cs +++ b/MediaBrowser.Server.Implementations/IO/FileRefresher.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using CommonIO; +using MediaBrowser.Common.Events; using MediaBrowser.Common.ScheduledTasks; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -24,9 +25,14 @@ namespace MediaBrowser.Server.Implementations.IO private readonly List _affectedPaths = new List(); private Timer _timer; private readonly object _timerLock = new object(); + public string Path { get; private set; } + + public event EventHandler Completed; public FileRefresher(string path, IFileSystem fileSystem, IServerConfigurationManager configurationManager, ILibraryManager libraryManager, ITaskManager taskManager, ILogger logger) { + logger.Debug("New file refresher created for {0}", path); + Path = path; _affectedPaths.Add(path); _fileSystem = fileSystem; @@ -36,7 +42,24 @@ namespace MediaBrowser.Server.Implementations.IO Logger = logger; } - private void RestartTimer() + private void AddAffectedPath(string path) + { + if (!_affectedPaths.Contains(path, StringComparer.Ordinal)) + { + _affectedPaths.Add(path); + } + } + + public void AddPath(string path) + { + lock (_timerLock) + { + AddAffectedPath(path); + } + RestartTimer(); + } + + public void RestartTimer() { lock (_timerLock) { @@ -51,6 +74,23 @@ namespace MediaBrowser.Server.Implementations.IO } } + public void ResetPath(string path, string affectedFile) + { + lock (_timerLock) + { + Logger.Debug("Resetting file refresher from {0} to {1}", Path, path); + + Path = path; + AddAffectedPath(path); + + if (!string.IsNullOrWhiteSpace(affectedFile)) + { + AddAffectedPath(affectedFile); + } + } + RestartTimer(); + } + private async void OnTimerCallback(object state) { // Extend the timer as long as any of the paths are still being written to. @@ -64,10 +104,11 @@ namespace MediaBrowser.Server.Implementations.IO Logger.Debug("Timer stopped."); DisposeTimer(); + EventHelper.FireEventIfNotNull(Completed, this, EventArgs.Empty, Logger); try { - await ProcessPathChanges(_affectedPaths).ConfigureAwait(false); + await ProcessPathChanges(_affectedPaths.ToList()).ConfigureAwait(false); } catch (Exception ex) { @@ -130,7 +171,7 @@ namespace MediaBrowser.Server.Implementations.IO { item = LibraryManager.FindByPath(path, null); - path = Path.GetDirectoryName(path); + path = System.IO.Path.GetDirectoryName(path); } if (item != null) @@ -222,7 +263,7 @@ namespace MediaBrowser.Server.Implementations.IO } } - public void DisposeTimer() + private void DisposeTimer() { lock (_timerLock) { diff --git a/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs b/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs index 09ca134d1..0de6ff306 100644 --- a/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs +++ b/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs @@ -26,13 +26,9 @@ namespace MediaBrowser.Server.Implementations.IO /// private readonly ConcurrentDictionary _fileSystemWatchers = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); /// - /// The update timer - /// - private Timer _updateTimer; - /// /// The affected paths /// - private readonly ConcurrentDictionary _affectedPaths = new ConcurrentDictionary(); + private readonly List _activeRefreshers = new List(); /// /// A dynamic list of paths that should be ignored. Added to during our own file sytem modifications. @@ -44,8 +40,8 @@ namespace MediaBrowser.Server.Implementations.IO /// private readonly IReadOnlyList _alwaysIgnoreFiles = new List { - "thumbs.db", - "small.jpg", + "thumbs.db", + "small.jpg", "albumart.jpg", // WMC temp recording directories that will constantly be written to @@ -53,11 +49,6 @@ namespace MediaBrowser.Server.Implementations.IO "TempSBE" }; - /// - /// The timer lock - /// - private readonly object _timerLock = new object(); - /// /// Add the path to our temporary ignore list. Use when writing to a path within our listening scope. /// @@ -463,226 +454,58 @@ namespace MediaBrowser.Server.Implementations.IO if (monitorPath) { // Avoid implicitly captured closure - var affectedPath = path; - _affectedPaths.AddOrUpdate(path, path, (key, oldValue) => affectedPath); - } - - RestartTimer(); - } - - private void RestartTimer() - { - lock (_timerLock) - { - if (_updateTimer == null) - { - _updateTimer = new Timer(TimerStopped, null, TimeSpan.FromSeconds(ConfigurationManager.Configuration.LibraryMonitorDelay), TimeSpan.FromMilliseconds(-1)); - } - else - { - _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.LibraryMonitorDelay), TimeSpan.FromMilliseconds(-1)); - } + CreateRefresher(path); } } - /// - /// Timers the stopped. - /// - /// The state info. - private async void TimerStopped(object stateInfo) + private void CreateRefresher(string path) { - // Extend the timer as long as any of the paths are still being written to. - if (_affectedPaths.Any(p => IsFileLocked(p.Key))) + var parentPath = Path.GetDirectoryName(path); + + lock (_activeRefreshers) { - Logger.Info("Timer extended."); - RestartTimer(); - return; - } - - Logger.Debug("Timer stopped."); - - DisposeTimer(); - - var paths = _affectedPaths.Keys.ToList(); - _affectedPaths.Clear(); - - try - { - await ProcessPathChanges(paths).ConfigureAwait(false); - } - catch (Exception ex) - { - Logger.ErrorException("Error processing directory changes", ex); - } - } - - private bool IsFileLocked(string path) - { - if (Environment.OSVersion.Platform != PlatformID.Win32NT) - { - // Causing lockups on linux - return false; - } - - try - { - var data = _fileSystem.GetFileSystemInfo(path); - - if (!data.Exists - || data.IsDirectory - - // Opening a writable stream will fail with readonly files - || data.Attributes.HasFlag(FileAttributes.ReadOnly)) + var refreshers = _activeRefreshers.ToList(); + foreach (var refresher in refreshers) { - return false; - } - } - catch (IOException) - { - return false; - } - catch (Exception ex) - { - Logger.ErrorException("Error getting file system info for: {0}", ex, path); - return false; - } - - // In order to determine if the file is being written to, we have to request write access - // But if the server only has readonly access, this is going to cause this entire algorithm to fail - // So we'll take a best guess about our access level - var requestedFileAccess = ConfigurationManager.Configuration.SaveLocalMeta - ? FileAccess.ReadWrite - : FileAccess.Read; - - try - { - using (_fileSystem.GetFileStream(path, FileMode.Open, requestedFileAccess, FileShare.ReadWrite)) - { - if (_updateTimer != null) + // Path is already being refreshed + if (string.Equals(path, refresher.Path, StringComparison.Ordinal)) { - //file is not locked - return false; + refresher.RestartTimer(); + return; + } + + // Parent folder is already being refreshed + if (_fileSystem.ContainsSubPath(refresher.Path, path)) + { + refresher.AddPath(path); + return; + } + + // New path is a parent + if (_fileSystem.ContainsSubPath(path, refresher.Path)) + { + refresher.ResetPath(path, null); + return; + } + + // Siblings + if (string.Equals(parentPath, Path.GetDirectoryName(refresher.Path), StringComparison.Ordinal)) + { + refresher.ResetPath(parentPath, path); + return; } } - } - catch (DirectoryNotFoundException) - { - // File may have been deleted - return false; - } - catch (FileNotFoundException) - { - // File may have been deleted - return false; - } - catch (IOException) - { - //the file is unavailable because it is: - //still being written to - //or being processed by another thread - //or does not exist (has already been processed) - Logger.Debug("{0} is locked.", path); - return true; - } - catch (Exception ex) - { - Logger.ErrorException("Error determining if file is locked: {0}", ex, path); - return false; - } - return false; - } - - private void DisposeTimer() - { - lock (_timerLock) - { - if (_updateTimer != null) - { - _updateTimer.Dispose(); - _updateTimer = null; - } + var newRefresher = new FileRefresher(path, _fileSystem, ConfigurationManager, LibraryManager, TaskManager, Logger); + newRefresher.Completed += NewRefresher_Completed; + _activeRefreshers.Add(newRefresher); } } - /// - /// Processes the path changes. - /// - /// The paths. - /// Task. - private async Task ProcessPathChanges(List paths) + private void NewRefresher_Completed(object sender, EventArgs e) { - var itemsToRefresh = paths - .Select(GetAffectedBaseItem) - .Where(item => item != null) - .Distinct() - .ToList(); - - foreach (var p in paths) - { - Logger.Info(p + " reports change."); - } - - // If the root folder changed, run the library task so the user can see it - if (itemsToRefresh.Any(i => i is AggregateFolder)) - { - TaskManager.CancelIfRunningAndQueue(); - return; - } - - foreach (var item in itemsToRefresh) - { - Logger.Info(item.Name + " (" + item.Path + ") will be refreshed."); - - try - { - await item.ChangedExternally().ConfigureAwait(false); - } - catch (IOException ex) - { - // For now swallow and log. - // Research item: If an IOException occurs, the item may be in a disconnected state (media unavailable) - // Should we remove it from it's parent? - Logger.ErrorException("Error refreshing {0}", ex, item.Name); - } - catch (Exception ex) - { - Logger.ErrorException("Error refreshing {0}", ex, item.Name); - } - } - } - - /// - /// Gets the affected base item. - /// - /// The path. - /// BaseItem. - private BaseItem GetAffectedBaseItem(string path) - { - BaseItem item = null; - - while (item == null && !string.IsNullOrEmpty(path)) - { - item = LibraryManager.FindByPath(path, null); - - path = Path.GetDirectoryName(path); - } - - if (item != null) - { - // If the item has been deleted find the first valid parent that still exists - while (!_fileSystem.DirectoryExists(item.Path) && !_fileSystem.FileExists(item.Path)) - { - item = item.GetParent(); - - if (item == null) - { - break; - } - } - } - - return item; + var refresher = (FileRefresher)sender; + DisposeRefresher(refresher); } /// @@ -713,10 +536,29 @@ namespace MediaBrowser.Server.Implementations.IO watcher.Dispose(); } - DisposeTimer(); - _fileSystemWatchers.Clear(); - _affectedPaths.Clear(); + DisposeRefreshers(); + } + + private void DisposeRefresher(FileRefresher refresher) + { + lock (_activeRefreshers) + { + refresher.Dispose(); + _activeRefreshers.Remove(refresher); + } + } + + private void DisposeRefreshers() + { + lock (_activeRefreshers) + { + foreach (var refresher in _activeRefreshers.ToList()) + { + refresher.Dispose(); + } + _activeRefreshers.Clear(); + } } ///