jellyfin/Emby.Server.Implementations/IO/LibraryMonitor.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Plugins;
using MediaBrowser.Model.IO;
using MediaBrowser.Model.System;
using Microsoft.Extensions.Logging;
using OperatingSystem = MediaBrowser.Common.System.OperatingSystem;

namespace Emby.Server.Implementations.IO
{
    public class LibraryMonitor : ILibraryMonitor
    {
        /// <summary>
        /// The file system watchers
        /// </summary>
        private readonly ConcurrentDictionary<string, FileSystemWatcher> _fileSystemWatchers = new ConcurrentDictionary<string, FileSystemWatcher>(StringComparer.OrdinalIgnoreCase);
        /// <summary>
        /// The affected paths
        /// </summary>
        private readonly List<FileRefresher> _activeRefreshers = new List<FileRefresher>();

        /// <summary>
        /// A dynamic list of paths that should be ignored.  Added to during our own file sytem modifications.
        /// </summary>
        private readonly ConcurrentDictionary<string, string> _tempIgnoredPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);

        /// <summary>
        /// Any file name ending in any of these will be ignored by the watchers
        /// </summary>
        private static readonly HashSet<string> _alwaysIgnoreFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
        {
            "small.jpg",
            "albumart.jpg",

            // WMC temp recording directories that will constantly be written to
            "TempRec",
            "TempSBE"
        };

        private static readonly string[] _alwaysIgnoreSubstrings = new string[]
        {
            // Synology
            "eaDir",
            "#recycle",
            ".wd_tv",
            ".actors"
        };

        private static readonly HashSet<string> _alwaysIgnoreExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
        {
            // thumbs.db
            ".db",

            // bts sync files
            ".bts",
            ".sync"
        };

        /// <summary>
        /// Add the path to our temporary ignore list.  Use when writing to a path within our listening scope.
        /// </summary>
        /// <param name="path">The path.</param>
        private void TemporarilyIgnore(string path)
        {
            _tempIgnoredPaths[path] = path;
        }

        public void ReportFileSystemChangeBeginning(string path)
        {
            if (string.IsNullOrEmpty(path))
            {
                throw new ArgumentNullException(nameof(path));
            }

            TemporarilyIgnore(path);
        }

        public bool IsPathLocked(string path)
        {
            // This method is not used by the core but it used by auto-organize

            var lockedPaths = _tempIgnoredPaths.Keys.ToList();
            return lockedPaths.Any(i => _fileSystem.AreEqual(i, path) || _fileSystem.ContainsSubPath(i, path));
        }

        public async void ReportFileSystemChangeComplete(string path, bool refreshPath)
        {
            if (string.IsNullOrEmpty(path))
            {
                throw new ArgumentNullException(nameof(path));
            }

            // This is an arbitraty amount of time, but delay it because file system writes often trigger events long after the file was actually written to.
            // Seeing long delays in some situations, especially over the network, sometimes up to 45 seconds
            // But if we make this delay too high, we risk missing legitimate changes, such as user adding a new file, or hand-editing metadata
            await Task.Delay(45000).ConfigureAwait(false);

            _tempIgnoredPaths.TryRemove(path, out var val);

            if (refreshPath)
            {
                try
                {
                    ReportFileSystemChanged(path);
                }
                catch (Exception ex)
                {
                    Logger.LogError(ex, "Error in ReportFileSystemChanged for {path}", path);
                }
            }
        }

        /// <summary>
        /// Gets or sets the logger.
        /// </summary>
        /// <value>The logger.</value>
        private ILogger Logger { get; set; }

        private ILibraryManager LibraryManager { get; set; }
        private IServerConfigurationManager ConfigurationManager { get; set; }

        private readonly IFileSystem _fileSystem;

        /// <summary>
        /// Initializes a new instance of the <see cref="LibraryMonitor" /> class.
        /// </summary>
        public LibraryMonitor(
            ILoggerFactory loggerFactory,
            ILibraryManager libraryManager,
            IServerConfigurationManager configurationManager,
            IFileSystem fileSystem)
        {
            LibraryManager = libraryManager;
            Logger = loggerFactory.CreateLogger(GetType().Name);
            ConfigurationManager = configurationManager;
            _fileSystem = fileSystem;
        }

        private bool IsLibraryMonitorEnabled(BaseItem item)
        {
            if (item is BasePluginFolder)
            {
                return false;
            }

            var options = LibraryManager.GetLibraryOptions(item);

            if (options != null)
            {
                return options.EnableRealtimeMonitor;
            }

            return false;
        }

        public void Start()
        {
            LibraryManager.ItemAdded += LibraryManager_ItemAdded;
            LibraryManager.ItemRemoved += LibraryManager_ItemRemoved;

            var pathsToWatch = new List<string> { };

            var paths = LibraryManager
                .RootFolder
                .Children
                .Where(IsLibraryMonitorEnabled)
                .OfType<Folder>()
                .SelectMany(f => f.PhysicalLocations)
                .Distinct(StringComparer.OrdinalIgnoreCase)
                .OrderBy(i => i)
                .ToList();

            foreach (var path in paths)
            {
                if (!ContainsParentFolder(pathsToWatch, path))
                {
                    pathsToWatch.Add(path);
                }
            }

            foreach (var path in pathsToWatch)
            {
                StartWatchingPath(path);
            }
        }

        private void StartWatching(BaseItem item)
        {
            if (IsLibraryMonitorEnabled(item))
            {
                StartWatchingPath(item.Path);
            }
        }

        /// <summary>
        /// Handles the ItemRemoved event of the LibraryManager control.
        /// </summary>
        /// <param name="sender">The source of the event.</param>
        /// <param name="e">The <see cref="ItemChangeEventArgs"/> instance containing the event data.</param>
        void LibraryManager_ItemRemoved(object sender, ItemChangeEventArgs e)
        {
            if (e.Parent is AggregateFolder)
            {
                StopWatchingPath(e.Item.Path);
            }
        }

        /// <summary>
        /// Handles the ItemAdded event of the LibraryManager control.
        /// </summary>
        /// <param name="sender">The source of the event.</param>
        /// <param name="e">The <see cref="ItemChangeEventArgs"/> instance containing the event data.</param>
        void LibraryManager_ItemAdded(object sender, ItemChangeEventArgs e)
        {
            if (e.Parent is AggregateFolder)
            {
                StartWatching(e.Item);
            }
        }

        /// <summary>
        /// Examine a list of strings assumed to be file paths to see if it contains a parent of
        /// the provided path.
        /// </summary>
        /// <param name="lst">The LST.</param>
        /// <param name="path">The path.</param>
        /// <returns><c>true</c> if [contains parent folder] [the specified LST]; otherwise, <c>false</c>.</returns>
        /// <exception cref="ArgumentNullException">path</exception>
        private static bool ContainsParentFolder(IEnumerable<string> lst, string path)
        {
            if (string.IsNullOrEmpty(path))
            {
                throw new ArgumentNullException(nameof(path));
            }

            path = path.TrimEnd(Path.DirectorySeparatorChar);

            return lst.Any(str =>
            {
                //this should be a little quicker than examining each actual parent folder...
                var compare = str.TrimEnd(Path.DirectorySeparatorChar);

                return path.Equals(compare, StringComparison.OrdinalIgnoreCase) || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == Path.DirectorySeparatorChar);
            });
        }

        /// <summary>
        /// Starts the watching path.
        /// </summary>
        /// <param name="path">The path.</param>
        private void StartWatchingPath(string path)
        {
            if (!Directory.Exists(path))
            {
                // Seeing a crash in the mono runtime due to an exception being thrown on a different thread
                Logger.LogInformation("Skipping realtime monitor for {0} because the path does not exist", path);
                return;
            }

            if (OperatingSystem.Id != OperatingSystemId.Windows)
            {
                if (path.StartsWith("\\\\", StringComparison.OrdinalIgnoreCase) || path.StartsWith("smb://", StringComparison.OrdinalIgnoreCase))
                {
                    // not supported
                    return;
                }
            }

            // Already being watched
            if (_fileSystemWatchers.ContainsKey(path))
            {
                return;
            }

            // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel
            Task.Run(() =>
            {
                try
                {
                    var newWatcher = new FileSystemWatcher(path, "*")
                    {
                        IncludeSubdirectories = true
                    };

                    newWatcher.InternalBufferSize = 65536;

                    newWatcher.NotifyFilter = NotifyFilters.CreationTime |
                        NotifyFilters.DirectoryName |
                        NotifyFilters.FileName |
                        NotifyFilters.LastWrite |
                        NotifyFilters.Size |
                        NotifyFilters.Attributes;

                    newWatcher.Created += watcher_Changed;
                    newWatcher.Deleted += watcher_Changed;
                    newWatcher.Renamed += watcher_Changed;
                    newWatcher.Changed += watcher_Changed;
                    newWatcher.Error += watcher_Error;

                    if (_fileSystemWatchers.TryAdd(path, newWatcher))
                    {
                        newWatcher.EnableRaisingEvents = true;
                        Logger.LogInformation("Watching directory " + path);
                    }
                    else
                    {
                        DisposeWatcher(newWatcher, false);
                    }

                }
                catch (Exception ex)
                {
                    Logger.LogError(ex, "Error watching path: {path}", path);
                }
            });
        }

        /// <summary>
        /// Stops the watching path.
        /// </summary>
        /// <param name="path">The path.</param>
        private void StopWatchingPath(string path)
        {
            if (_fileSystemWatchers.TryGetValue(path, out var watcher))
            {
                DisposeWatcher(watcher, true);
            }
        }

        /// <summary>
        /// Disposes the watcher.
        /// </summary>
        private void DisposeWatcher(FileSystemWatcher watcher, bool removeFromList)
        {
            try
            {
                using (watcher)
                {
                    Logger.LogInformation("Stopping directory watching for path {path}", watcher.Path);

                    watcher.Created -= watcher_Changed;
                    watcher.Deleted -= watcher_Changed;
                    watcher.Renamed -= watcher_Changed;
                    watcher.Changed -= watcher_Changed;
                    watcher.Error -= watcher_Error;

                    try
                    {
                        watcher.EnableRaisingEvents = false;
                    }
                    catch (InvalidOperationException)
                    {
                        // Seeing this under mono on linux sometimes
                        // Collection was modified; enumeration operation may not execute.
                    }
                }
            }
            catch (NotImplementedException)
            {
                // the dispose method on FileSystemWatcher is sometimes throwing NotImplementedException on Xamarin Android
            }
            catch
            {

            }
            finally
            {
                if (removeFromList)
                {
                    RemoveWatcherFromList(watcher);
                }
            }
        }

        /// <summary>
        /// Removes the watcher from list.
        /// </summary>
        /// <param name="watcher">The watcher.</param>
        private void RemoveWatcherFromList(FileSystemWatcher watcher)
        {
            _fileSystemWatchers.TryRemove(watcher.Path, out var removed);
        }

        /// <summary>
        /// Handles the Error event of the watcher control.
        /// </summary>
        /// <param name="sender">The source of the event.</param>
        /// <param name="e">The <see cref="ErrorEventArgs" /> instance containing the event data.</param>
        void watcher_Error(object sender, ErrorEventArgs e)
        {
            var ex = e.GetException();
            var dw = (FileSystemWatcher)sender;

            Logger.LogError(ex, "Error in Directory watcher for: {path}", dw.Path);

            DisposeWatcher(dw, true);
        }

        /// <summary>
        /// Handles the Changed event of the watcher control.
        /// </summary>
        /// <param name="sender">The source of the event.</param>
        /// <param name="e">The <see cref="FileSystemEventArgs" /> instance containing the event data.</param>
        void watcher_Changed(object sender, FileSystemEventArgs e)
        {
            try
            {
                //logger.LogDebug("Changed detected of type " + e.ChangeType + " to " + e.FullPath);

                var path = e.FullPath;

                ReportFileSystemChanged(path);
            }
            catch (Exception ex)
            {
                Logger.LogError(ex, "Exception in ReportFileSystemChanged. Path: {FullPath}", e.FullPath);
            }
        }

        public void ReportFileSystemChanged(string path)
        {
            if (string.IsNullOrEmpty(path))
            {
                throw new ArgumentNullException(nameof(path));
            }

            var filename = Path.GetFileName(path);

            var monitorPath = !string.IsNullOrEmpty(filename) &&
                !_alwaysIgnoreFiles.Contains(filename) &&
                !_alwaysIgnoreExtensions.Contains(Path.GetExtension(path)) &&
                _alwaysIgnoreSubstrings.All(i => path.IndexOf(i, StringComparison.OrdinalIgnoreCase) == -1);

            // Ignore certain files
            var tempIgnorePaths = _tempIgnoredPaths.Keys.ToList();

            // If the parent of an ignored path has a change event, ignore that too
            if (tempIgnorePaths.Any(i =>
            {
                if (_fileSystem.AreEqual(i, path))
                {
                    Logger.LogDebug("Ignoring change to {path}", path);
                    return true;
                }

                if (_fileSystem.ContainsSubPath(i, path))
                {
                    Logger.LogDebug("Ignoring change to {path}", path);
                    return true;
                }

                // Go up a level
                var parent = Path.GetDirectoryName(i);
                if (!string.IsNullOrEmpty(parent))
                {
                    if (_fileSystem.AreEqual(parent, path))
                    {
                        Logger.LogDebug("Ignoring change to {path}", path);
                        return true;
                    }
                }

                return false;

            }))
            {
                monitorPath = false;
            }

            if (monitorPath)
            {
                // Avoid implicitly captured closure
                CreateRefresher(path);
            }
        }

        private void CreateRefresher(string path)
        {
            var parentPath = Path.GetDirectoryName(path);

            lock (_activeRefreshers)
            {
                var refreshers = _activeRefreshers.ToList();
                foreach (var refresher in refreshers)
                {
                    // Path is already being refreshed
                    if (_fileSystem.AreEqual(path, refresher.Path))
                    {
                        refresher.RestartTimer();
                        return;
                    }

                    // Parent folder is already being refreshed
                    if (_fileSystem.ContainsSubPath(refresher.Path, path))
                    {
                        refresher.AddPath(path);
                        return;
                    }

                    // New path is a parent
                    if (_fileSystem.ContainsSubPath(path, refresher.Path))
                    {
                        refresher.ResetPath(path, null);
                        return;
                    }

                    // They are siblings. Rebase the refresher to the parent folder.
                    if (string.Equals(parentPath, Path.GetDirectoryName(refresher.Path), StringComparison.Ordinal))
                    {
                        refresher.ResetPath(parentPath, path);
                        return;
                    }
                }

                var newRefresher = new FileRefresher(path, ConfigurationManager, LibraryManager, Logger);
                newRefresher.Completed += NewRefresher_Completed;
                _activeRefreshers.Add(newRefresher);
            }
        }

        private void NewRefresher_Completed(object sender, EventArgs e)
        {
            var refresher = (FileRefresher)sender;
            DisposeRefresher(refresher);
        }

        /// <summary>
        /// Stops this instance.
        /// </summary>
        public void Stop()
        {
            LibraryManager.ItemAdded -= LibraryManager_ItemAdded;
            LibraryManager.ItemRemoved -= LibraryManager_ItemRemoved;

            foreach (var watcher in _fileSystemWatchers.Values.ToList())
            {
                DisposeWatcher(watcher, false);
            }

            _fileSystemWatchers.Clear();
            DisposeRefreshers();
        }

        private void DisposeRefresher(FileRefresher refresher)
        {
            lock (_activeRefreshers)
            {
                refresher.Dispose();
                _activeRefreshers.Remove(refresher);
            }
        }

        private void DisposeRefreshers()
        {
            lock (_activeRefreshers)
            {
                foreach (var refresher in _activeRefreshers.ToList())
                {
                    refresher.Dispose();
                }
                _activeRefreshers.Clear();
            }
        }

        private bool _disposed;
        /// <summary>
        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
        /// </summary>
        public void Dispose()
        {
            Dispose(true);
        }

        /// <summary>
        /// Releases unmanaged and - optionally - managed resources.
        /// </summary>
        /// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
        protected virtual void Dispose(bool disposing)
        {
            if (_disposed)
            {
                return;
            }

            if (disposing)
            {
                Stop();
            }

            _disposed = true;
        }
    }

    public class LibraryMonitorStartup : IServerEntryPoint
    {
        private readonly ILibraryMonitor _monitor;

        public LibraryMonitorStartup(ILibraryMonitor monitor)
        {
            _monitor = monitor;
        }

        public Task RunAsync()
        {
            _monitor.Start();
            return Task.CompletedTask;
        }

        public void Dispose()
        {
        }
    }
}