using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Providers { /// /// Class ProviderManager /// public class ProviderManager : IProviderManager { /// /// The remote image cache /// private readonly FileSystemRepository _remoteImageCache; /// /// The currently running metadata providers /// private readonly ConcurrentDictionary> _currentlyRunningProviders = new ConcurrentDictionary>(); /// /// The _logger /// private readonly ILogger _logger; /// /// The _HTTP client /// private readonly IHttpClient _httpClient; /// /// The _directory watchers /// private readonly IDirectoryWatchers _directoryWatchers; /// /// Gets or sets the configuration manager. /// /// The configuration manager. private IServerConfigurationManager ConfigurationManager { get; set; } /// /// Gets the list of currently registered metadata prvoiders /// /// The metadata providers enumerable. private BaseMetadataProvider[] MetadataProviders { get; set; } private IEnumerable _savers; /// /// Initializes a new instance of the class. /// /// The HTTP client. /// The configuration manager. /// The directory watchers. /// The log manager. /// The library manager. public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager, ILibraryManager libraryManager) { _logger = logManager.GetLogger("ProviderManager"); _httpClient = httpClient; ConfigurationManager = configurationManager; _directoryWatchers = directoryWatchers; _remoteImageCache = new FileSystemRepository(configurationManager.ApplicationPaths.DownloadedImagesDataPath); configurationManager.ConfigurationUpdated += configurationManager_ConfigurationUpdated; libraryManager.ItemUpdated += libraryManager_ItemUpdated; } /// /// Handles the ItemUpdated event of the libraryManager control. /// /// The source of the event. /// The instance containing the event data. void libraryManager_ItemUpdated(object sender, ItemChangeEventArgs e) { var item = e.Item; if (ConfigurationManager.Configuration.SaveLocalMeta) { if (item.LocationType != LocationType.FileSystem) { return; } foreach (var saver in _savers.Where(i => i.Supports(item))) { var path = saver.GetSavePath(item); _directoryWatchers.TemporarilyIgnore(path); try { saver.Save(item, CancellationToken.None); } catch (Exception ex) { _logger.ErrorException("Error in metadata saver", ex); } finally { _directoryWatchers.RemoveTempIgnore(path); } } } } /// /// Handles the ConfigurationUpdated event of the configurationManager control. /// /// The source of the event. /// The instance containing the event data. void configurationManager_ConfigurationUpdated(object sender, EventArgs e) { // Validate currently executing providers, in the background Task.Run(() => ValidateCurrentlyRunningProviders()); } /// /// Adds the metadata providers. /// /// The providers. /// The savers. public void AddParts(IEnumerable providers, IEnumerable savers) { MetadataProviders = providers.OrderBy(e => e.Priority).ToArray(); _savers = savers; } /// /// Runs all metadata providers for an entity, and returns true or false indicating if at least one was refreshed and requires persistence /// /// The item. /// The cancellation token. /// if set to true [force]. /// if set to true [allow slow providers]. /// Task{System.Boolean}. public async Task ExecuteMetadataProviders(BaseItem item, CancellationToken cancellationToken, bool force = false, bool allowSlowProviders = true) { if (item == null) { throw new ArgumentNullException("item"); } // Allow providers of the same priority to execute in parallel MetadataProviderPriority? currentPriority = null; var currentTasks = new List>(); var result = false; cancellationToken.ThrowIfCancellationRequested(); // Run the normal providers sequentially in order of priority foreach (var provider in MetadataProviders.Where(p => p.Supports(item))) { cancellationToken.ThrowIfCancellationRequested(); // Skip if internet providers are currently disabled if (provider.RequiresInternet && !ConfigurationManager.Configuration.EnableInternetProviders) { continue; } // Skip if is slow and we aren't allowing slow ones if (provider.IsSlow && !allowSlowProviders) { continue; } // Skip if internet provider and this type is not allowed if (provider.RequiresInternet && ConfigurationManager.Configuration.EnableInternetProviders && ConfigurationManager.Configuration.InternetProviderExcludeTypes.Contains(item.GetType().Name, StringComparer.OrdinalIgnoreCase)) { continue; } // When a new priority is reached, await the ones that are currently running and clear the list if (currentPriority.HasValue && currentPriority.Value != provider.Priority && currentTasks.Count > 0) { var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); result |= results.Contains(true); currentTasks.Clear(); } // Put this check below the await because the needs refresh of the next tier of providers may depend on the previous ones running // This is the case for the fan art provider which depends on the movie and tv providers having run before them if (provider.RequiresInternet && item.DontFetchMeta) { continue; } try { if (!force && !provider.NeedsRefresh(item)) { continue; } } catch (Exception ex) { _logger.Error("Error determining NeedsRefresh for {0}", ex, item.Path); } currentTasks.Add(FetchAsync(provider, item, force, cancellationToken)); currentPriority = provider.Priority; } if (currentTasks.Count > 0) { var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); result |= results.Contains(true); } return result; } /// /// Fetches metadata and returns true or false indicating if any work that requires persistence was done /// /// The provider. /// The item. /// if set to true [force]. /// The cancellation token. /// Task{System.Boolean}. /// private async Task FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken) { if (item == null) { throw new ArgumentNullException(); } cancellationToken.ThrowIfCancellationRequested(); _logger.Debug("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name ?? "--Unknown--"); // This provides the ability to cancel just this one provider var innerCancellationTokenSource = new CancellationTokenSource(); OnProviderRefreshBeginning(provider, item, innerCancellationTokenSource); try { return await provider.FetchAsync(item, force, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, innerCancellationTokenSource.Token).Token).ConfigureAwait(false); } catch (OperationCanceledException ex) { _logger.Debug("{0} canceled for {1}", provider.GetType().Name, item.Name); // If the outer cancellation token is the one that caused the cancellation, throw it if (cancellationToken.IsCancellationRequested && ex.CancellationToken == cancellationToken) { throw; } return false; } catch (Exception ex) { _logger.ErrorException("{0} failed refreshing {1}", ex, provider.GetType().Name, item.Name); provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure); return true; } finally { innerCancellationTokenSource.Dispose(); OnProviderRefreshCompleted(provider, item); } } /// /// Notifies the kernal that a provider has begun refreshing /// /// The provider. /// The item. /// The cancellation token source. public void OnProviderRefreshBeginning(BaseMetadataProvider provider, BaseItem item, CancellationTokenSource cancellationTokenSource) { var key = item.Id + provider.GetType().Name; Tuple current; if (_currentlyRunningProviders.TryGetValue(key, out current)) { try { current.Item3.Cancel(); } catch (ObjectDisposedException) { } } var tuple = new Tuple(provider, item, cancellationTokenSource); _currentlyRunningProviders.AddOrUpdate(key, tuple, (k, v) => tuple); } /// /// Notifies the kernal that a provider has completed refreshing /// /// The provider. /// The item. public void OnProviderRefreshCompleted(BaseMetadataProvider provider, BaseItem item) { var key = item.Id + provider.GetType().Name; Tuple current; if (_currentlyRunningProviders.TryRemove(key, out current)) { current.Item3.Dispose(); } } /// /// Validates the currently running providers and cancels any that should not be run due to configuration changes /// private void ValidateCurrentlyRunningProviders() { var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders; var internetProviderExcludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes; foreach (var tuple in _currentlyRunningProviders.Values .Where(p => p.Item1.RequiresInternet && (!enableInternetProviders || internetProviderExcludeTypes.Contains(p.Item2.GetType().Name, StringComparer.OrdinalIgnoreCase))) .ToList()) { tuple.Item3.Cancel(); } } /// /// Downloads the and save image. /// /// The item. /// The source. /// Name of the target. /// if set to true [save locally]. /// The resource pool. /// The cancellation token. /// Task{System.String}. /// item public async Task DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { if (item == null) { throw new ArgumentNullException("item"); } if (string.IsNullOrEmpty(source)) { throw new ArgumentNullException("source"); } if (string.IsNullOrEmpty(targetName)) { throw new ArgumentNullException("targetName"); } if (resourcePool == null) { throw new ArgumentNullException("resourcePool"); } var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false); //download and save locally return await SaveImage(item, img, targetName, saveLocally, cancellationToken).ConfigureAwait(false); } public async Task SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken) { //download and save locally var localPath = GetSavePath(item, targetName, saveLocally); if (saveLocally) // queue to media directories { await SaveToLibraryFilesystem(item, localPath, source, cancellationToken).ConfigureAwait(false); } else { // we can write directly here because it won't affect the watchers try { using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await source.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } } catch (OperationCanceledException) { throw; } catch (Exception e) { _logger.ErrorException("Error downloading and saving image " + localPath, e); throw; } finally { source.Dispose(); } } return localPath; } /// /// Gets the save path. /// /// The item. /// Name of the target file. /// if set to true [save locally]. /// System.String. public string GetSavePath(BaseItem item, string targetFileName, bool saveLocally) { var path = (saveLocally && item.MetaLocation != null) ? Path.Combine(item.MetaLocation, targetFileName) : _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Id.ToString(), targetFileName); var parentPath = Path.GetDirectoryName(path); if (!Directory.Exists(parentPath)) { Directory.CreateDirectory(parentPath); } return path; } /// /// Saves to library filesystem. /// /// The item. /// The path. /// The data to save. /// The cancellation token. /// Task. /// public async Task SaveToLibraryFilesystem(BaseItem item, string path, Stream dataToSave, CancellationToken cancellationToken) { if (item == null) { throw new ArgumentNullException(); } if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(); } if (dataToSave == null) { throw new ArgumentNullException(); } if (cancellationToken == null) { throw new ArgumentNullException(); } if (cancellationToken.IsCancellationRequested) { dataToSave.Dispose(); cancellationToken.ThrowIfCancellationRequested(); } //Tell the watchers to ignore _directoryWatchers.TemporarilyIgnore(path); if (dataToSave.CanSeek) { dataToSave.Position = 0; } try { using (dataToSave) { using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await dataToSave.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } } // If this is ever used for something other than metadata we can add a file type param item.ResolveArgs.AddMetadataFile(path); } finally { //Remove the ignore _directoryWatchers.RemoveTempIgnore(path); } } } }