using MediaBrowser.Common.Events;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.ScheduledTasks;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Movies;
using MediaBrowser.Controller.IO;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Controller.Resolvers;
using MediaBrowser.Controller.Sorting;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using MediaBrowser.Server.Implementations.ScheduledTasks;
using MoreLinq;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SortOrder = MediaBrowser.Model.Entities.SortOrder;
namespace MediaBrowser.Server.Implementations.Library
{
///
/// Class LibraryManager
///
public class LibraryManager : ILibraryManager
{
///
/// Gets the intro providers.
///
/// The intro providers.
private IEnumerable IntroProviders { get; set; }
///
/// Gets the list of entity resolution ignore rules
///
/// The entity resolution ignore rules.
private IEnumerable EntityResolutionIgnoreRules { get; set; }
///
/// Gets the list of BasePluginFolders added by plugins
///
/// The plugin folders.
private IEnumerable PluginFolderCreators { get; set; }
///
/// Gets the list of currently registered entity resolvers
///
/// The entity resolvers enumerable.
private IEnumerable EntityResolvers { get; set; }
///
/// Gets or sets the comparers.
///
/// The comparers.
private IEnumerable Comparers { get; set; }
///
/// Gets the active item repository
///
/// The item repository.
public IItemRepository ItemRepository { get; set; }
#region LibraryChanged Event
///
/// Fires whenever any validation routine adds or removes items. The added and removed items are properties of the args.
/// *** Will fire asynchronously. ***
///
public event EventHandler LibraryChanged;
///
/// Reports the library changed.
///
/// The instance containing the event data.
public void ReportLibraryChanged(ChildrenChangedEventArgs args)
{
UpdateLibraryCache(args);
EventHelper.FireEventIfNotNull(LibraryChanged, this, args, _logger);
}
#endregion
///
/// The _logger
///
private readonly ILogger _logger;
///
/// The _task manager
///
private readonly ITaskManager _taskManager;
///
/// The _user manager
///
private readonly IUserManager _userManager;
private readonly IUserDataRepository _userDataRepository;
///
/// Gets or sets the configuration manager.
///
/// The configuration manager.
private IServerConfigurationManager ConfigurationManager { get; set; }
///
/// A collection of items that may be referenced from multiple physical places in the library
/// (typically, multiple user roots). We store them here and be sure they all reference a
/// single instance.
///
private ConcurrentDictionary ByReferenceItems { get; set; }
private ConcurrentDictionary _libraryItemsCache;
private object _libraryItemsCacheSyncLock = new object();
private bool _libraryItemsCacheInitialized;
private ConcurrentDictionary LibraryItemsCache
{
get
{
LazyInitializer.EnsureInitialized(ref _libraryItemsCache, ref _libraryItemsCacheInitialized, ref _libraryItemsCacheSyncLock, CreateLibraryItemsCache);
return _libraryItemsCache;
}
}
private readonly ConcurrentDictionary _userRootFolders =
new ConcurrentDictionary();
///
/// Initializes a new instance of the class.
///
/// The logger.
/// The task manager.
/// The user manager.
/// The configuration manager.
/// The user data repository.
public LibraryManager(ILogger logger, ITaskManager taskManager, IUserManager userManager, IServerConfigurationManager configurationManager, IUserDataRepository userDataRepository)
{
_logger = logger;
_taskManager = taskManager;
_userManager = userManager;
ConfigurationManager = configurationManager;
_userDataRepository = userDataRepository;
ByReferenceItems = new ConcurrentDictionary();
ConfigurationManager.ConfigurationUpdated += ConfigurationUpdated;
RecordConfigurationValues(configurationManager.Configuration);
}
///
/// Adds the parts.
///
/// The rules.
/// The plugin folders.
/// The resolvers.
/// The intro providers.
/// The item comparers.
public void AddParts(IEnumerable rules,
IEnumerable pluginFolders,
IEnumerable resolvers,
IEnumerable introProviders,
IEnumerable itemComparers)
{
EntityResolutionIgnoreRules = rules;
PluginFolderCreators = pluginFolders;
EntityResolvers = resolvers.OrderBy(i => i.Priority).ToArray();
IntroProviders = introProviders;
Comparers = itemComparers;
}
///
/// The _root folder
///
private AggregateFolder _rootFolder;
///
/// The _root folder sync lock
///
private object _rootFolderSyncLock = new object();
///
/// The _root folder initialized
///
private bool _rootFolderInitialized;
///
/// Gets the root folder.
///
/// The root folder.
public AggregateFolder RootFolder
{
get
{
LazyInitializer.EnsureInitialized(ref _rootFolder, ref _rootFolderInitialized, ref _rootFolderSyncLock, CreateRootFolder);
return _rootFolder;
}
private set
{
_rootFolder = value;
if (value == null)
{
_rootFolderInitialized = false;
}
}
}
private bool _internetProvidersEnabled;
private bool _peopleImageFetchingEnabled;
private void RecordConfigurationValues(ServerConfiguration configuration)
{
_internetProvidersEnabled = configuration.EnableInternetProviders;
_peopleImageFetchingEnabled = configuration.InternetProviderExcludeTypes == null || !configuration.InternetProviderExcludeTypes.Contains(typeof(Person).Name, StringComparer.OrdinalIgnoreCase);
}
///
/// Configurations the updated.
///
/// The sender.
/// The instance containing the event data.
void ConfigurationUpdated(object sender, EventArgs e)
{
var config = ConfigurationManager.Configuration;
// Figure out whether or not we should refresh people after the update is finished
var refreshPeopleAfterUpdate = !_internetProvidersEnabled && config.EnableInternetProviders;
// This is true if internet providers has just been turned on, or if People have just been removed from InternetProviderExcludeTypes
if (!refreshPeopleAfterUpdate)
{
var newConfigurationFetchesPeopleImages = config.InternetProviderExcludeTypes == null || !config.InternetProviderExcludeTypes.Contains(typeof(Person).Name, StringComparer.OrdinalIgnoreCase);
refreshPeopleAfterUpdate = newConfigurationFetchesPeopleImages && !_peopleImageFetchingEnabled;
}
RecordConfigurationValues(config);
Task.Run(() =>
{
// Any number of configuration settings could change the way the library is refreshed, so do that now
_taskManager.CancelIfRunningAndQueue();
if (refreshPeopleAfterUpdate)
{
_taskManager.CancelIfRunningAndQueue();
}
});
}
///
/// Creates the library items cache.
///
/// ConcurrentDictionary{GuidBaseItem}.
private ConcurrentDictionary CreateLibraryItemsCache()
{
var items = RootFolder.RecursiveChildren.ToList();
items.Add(RootFolder);
var specialFeatures = items.OfType().SelectMany(i => i.SpecialFeatures).ToList();
var localTrailers = items.SelectMany(i => i.LocalTrailers).ToList();
items.AddRange(specialFeatures);
items.AddRange(localTrailers);
// Need to use DistinctBy Id because there could be multiple instances with the same id
// due to sharing the default library
var userRootFolders = _userManager.Users.Select(i => i.RootFolder)
.DistinctBy(i => i.Id)
.ToList();
items.AddRange(userRootFolders);
// Get all user collection folders
var userFolders =
_userManager.Users.SelectMany(i => i.RootFolder.Children)
.Where(i => !(i is BasePluginFolder))
.DistinctBy(i => i.Id)
.ToList();
items.AddRange(userFolders);
return new ConcurrentDictionary(items.ToDictionary(i => i.Id));
}
///
/// Updates the library cache.
///
/// The instance containing the event data.
private void UpdateLibraryCache(ChildrenChangedEventArgs args)
{
UpdateItemInLibraryCache(args.Folder);
foreach (var item in args.ItemsAdded)
{
UpdateItemInLibraryCache(item);
}
foreach (var item in args.ItemsUpdated)
{
UpdateItemInLibraryCache(item);
}
}
///
/// Updates the item in library cache.
///
/// The item.
private void UpdateItemInLibraryCache(BaseItem item)
{
LibraryItemsCache.AddOrUpdate(item.Id, item, delegate { return item; });
foreach (var trailer in item.LocalTrailers)
{
// Prevent access to foreach variable in closure
var trailer1 = trailer;
LibraryItemsCache.AddOrUpdate(trailer.Id, trailer, delegate { return trailer1; });
}
var movie = item as Movie;
if (movie != null)
{
foreach (var special in movie.SpecialFeatures)
{
// Prevent access to foreach variable in closure
var special1 = special;
LibraryItemsCache.AddOrUpdate(special.Id, special, delegate { return special1; });
}
}
}
///
/// Resolves the item.
///
/// The args.
/// BaseItem.
public BaseItem ResolveItem(ItemResolveArgs args)
{
var item = EntityResolvers.Select(r => r.ResolvePath(args)).FirstOrDefault(i => i != null);
if (item != null)
{
ResolverHelper.SetInitialItemValues(item, args);
// Now handle the issue with posibly having the same item referenced from multiple physical
// places within the library. Be sure we always end up with just one instance.
if (item is IByReferenceItem)
{
item = GetOrAddByReferenceItem(item);
}
}
return item;
}
///
/// Ensure supplied item has only one instance throughout
///
///
/// The proper instance to the item
public BaseItem GetOrAddByReferenceItem(BaseItem item)
{
// Add this item to our list if not there already
if (!ByReferenceItems.TryAdd(item.Id, item))
{
// Already there - return the existing reference
item = ByReferenceItems[item.Id];
}
return item;
}
///
/// Resolves a path into a BaseItem
///
/// The path.
/// The parent.
/// The file info.
/// BaseItem.
///
public BaseItem ResolvePath(string path, Folder parent = null, WIN32_FIND_DATA? fileInfo = null)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException();
}
fileInfo = fileInfo ?? FileSystem.GetFileData(path);
if (!fileInfo.HasValue)
{
return null;
}
var args = new ItemResolveArgs(ConfigurationManager.ApplicationPaths)
{
Parent = parent,
Path = path,
FileInfo = fileInfo.Value
};
// Return null if ignore rules deem that we should do so
if (EntityResolutionIgnoreRules.Any(r => r.ShouldIgnore(args)))
{
return null;
}
// Gather child folder and files
if (args.IsDirectory)
{
// When resolving the root, we need it's grandchildren (children of user views)
var flattenFolderDepth = args.IsPhysicalRoot ? 2 : 0;
args.FileSystemDictionary = FileData.GetFilteredFileSystemEntries(args.Path, _logger, flattenFolderDepth: flattenFolderDepth, args: args);
}
// Check to see if we should resolve based on our contents
if (args.IsDirectory && !ShouldResolvePathContents(args))
{
return null;
}
return ResolveItem(args);
}
///
/// Determines whether a path should be ignored based on its contents - called after the contents have been read
///
/// The args.
/// true if XXXX, false otherwise
private static bool ShouldResolvePathContents(ItemResolveArgs args)
{
// Ignore any folders containing a file called .ignore
return !args.ContainsFileSystemEntryByName(".ignore");
}
///
/// Resolves a set of files into a list of BaseItem
///
///
/// The files.
/// The parent.
/// List{``0}.
public List ResolvePaths(IEnumerable files, Folder parent)
where T : BaseItem
{
var list = new List();
Parallel.ForEach(files, f =>
{
try
{
var item = ResolvePath(f.Path, parent, f) as T;
if (item != null)
{
lock (list)
{
list.Add(item);
}
}
}
catch (Exception ex)
{
_logger.ErrorException("Error resolving path {0}", ex, f.Path);
}
});
return list;
}
///
/// Creates the root media folder
///
/// AggregateFolder.
/// Cannot create the root folder until plugins have loaded
public AggregateFolder CreateRootFolder()
{
var rootFolderPath = ConfigurationManager.ApplicationPaths.RootFolderPath;
var rootFolder = RetrieveItem(rootFolderPath.GetMBId(typeof(AggregateFolder))) as AggregateFolder ?? (AggregateFolder)ResolvePath(rootFolderPath);
// Add in the plug-in folders
foreach (var child in PluginFolderCreators)
{
rootFolder.AddVirtualChild(child.GetFolder());
}
return rootFolder;
}
///
/// Gets the user root folder.
///
/// The user root path.
/// UserRootFolder.
public UserRootFolder GetUserRootFolder(string userRootPath)
{
return _userRootFolders.GetOrAdd(userRootPath, key => RetrieveItem(userRootPath.GetMBId(typeof(UserRootFolder))) as UserRootFolder ?? (UserRootFolder)ResolvePath(userRootPath));
}
///
/// Gets a Person
///
/// The name.
/// if set to true [allow slow providers].
/// Task{Person}.
public Task GetPerson(string name, bool allowSlowProviders = false)
{
return GetPerson(name, CancellationToken.None, allowSlowProviders);
}
///
/// Gets a Person
///
/// The name.
/// The cancellation token.
/// if set to true [allow slow providers].
/// if set to true [force creation].
/// Task{Person}.
private Task GetPerson(string name, CancellationToken cancellationToken, bool allowSlowProviders = false, bool forceCreation = false)
{
return GetImagesByNameItem(ConfigurationManager.ApplicationPaths.PeoplePath, name, cancellationToken, allowSlowProviders, forceCreation);
}
///
/// Gets a Studio
///
/// The name.
/// if set to true [allow slow providers].
/// Task{Studio}.
public Task GetStudio(string name, bool allowSlowProviders = false)
{
return GetImagesByNameItem(ConfigurationManager.ApplicationPaths.StudioPath, name, CancellationToken.None, allowSlowProviders);
}
///
/// Gets a Genre
///
/// The name.
/// if set to true [allow slow providers].
/// Task{Genre}.
public Task GetGenre(string name, bool allowSlowProviders = false)
{
return GetImagesByNameItem(ConfigurationManager.ApplicationPaths.GenrePath, name, CancellationToken.None, allowSlowProviders);
}
///
/// The us culture
///
private static readonly CultureInfo UsCulture = new CultureInfo("en-US");
///
/// Gets a Year
///
/// The value.
/// if set to true [allow slow providers].
/// Task{Year}.
///
public Task GetYear(int value, bool allowSlowProviders = false)
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException();
}
return GetImagesByNameItem(ConfigurationManager.ApplicationPaths.YearPath, value.ToString(UsCulture), CancellationToken.None, allowSlowProviders);
}
///
/// The images by name item cache
///
private readonly ConcurrentDictionary _imagesByNameItemCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase);
///
/// Generically retrieves an IBN item
///
///
/// The path.
/// The name.
/// The cancellation token.
/// if set to true [allow slow providers].
/// if set to true [force creation].
/// Task{``0}.
///
///
private async Task GetImagesByNameItem(string path, string name, CancellationToken cancellationToken, bool allowSlowProviders = true, bool forceCreation = false)
where T : BaseItem, new()
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException();
}
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException();
}
var key = Path.Combine(path, FileSystem.GetValidFilename(name));
BaseItem obj;
if (forceCreation || !_imagesByNameItemCache.TryGetValue(key, out obj))
{
obj = await CreateImagesByNameItem(path, name, cancellationToken, allowSlowProviders).ConfigureAwait(false);
_imagesByNameItemCache.AddOrUpdate(key, obj, (keyName, oldValue) => obj);
}
return obj as T;
}
///
/// Creates an IBN item based on a given path
///
///
/// The path.
/// The name.
/// The cancellation token.
/// if set to true [allow slow providers].
/// Task{``0}.
/// Path not created: + path
private async Task CreateImagesByNameItem(string path, string name, CancellationToken cancellationToken, bool allowSlowProviders = true)
where T : BaseItem, new()
{
cancellationToken.ThrowIfCancellationRequested();
_logger.Debug("Getting {0}: {1}", typeof(T).Name, name);
path = Path.Combine(path, FileSystem.GetValidFilename(name));
var fileInfo = FileSystem.GetFileData(path);
var isNew = false;
if (!fileInfo.HasValue)
{
Directory.CreateDirectory(path);
fileInfo = FileSystem.GetFileData(path);
if (!fileInfo.HasValue)
{
throw new IOException("Path not created: " + path);
}
isNew = true;
}
cancellationToken.ThrowIfCancellationRequested();
var id = path.GetMBId(typeof(T));
var item = RetrieveItem(id) as T;
if (item == null)
{
item = new T
{
Name = name,
Id = id,
DateCreated = fileInfo.Value.CreationTimeUtc,
DateModified = fileInfo.Value.LastWriteTimeUtc,
Path = path
};
isNew = true;
}
cancellationToken.ThrowIfCancellationRequested();
// Set this now so we don't cause additional file system access during provider executions
item.ResetResolveArgs(fileInfo);
await item.RefreshMetadata(cancellationToken, isNew, allowSlowProviders: allowSlowProviders).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return item;
}
///
/// Validate and refresh the People sub-set of the IBN.
/// The items are stored in the db but not loaded into memory until actually requested by an operation.
///
/// The cancellation token.
/// The progress.
/// Task.
public async Task ValidatePeople(CancellationToken cancellationToken, IProgress progress)
{
const int maxTasks = 50;
var tasks = new List();
var includedPersonTypes = new[] { PersonType.Actor, PersonType.Director, PersonType.GuestStar, PersonType.Writer, PersonType.Director, PersonType.Producer };
var people = RootFolder.RecursiveChildren
.Where(c => c.People != null)
.SelectMany(c => c.People.Where(p => includedPersonTypes.Contains(p.Type)))
.DistinctBy(p => p.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
var numComplete = 0;
foreach (var person in people)
{
if (tasks.Count > maxTasks)
{
await Task.WhenAll(tasks).ConfigureAwait(false);
tasks.Clear();
// Safe cancellation point, when there are no pending tasks
cancellationToken.ThrowIfCancellationRequested();
}
// Avoid accessing the foreach variable within the closure
var currentPerson = person;
tasks.Add(Task.Run(async () =>
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await GetPerson(currentPerson.Name, cancellationToken, true, true).ConfigureAwait(false);
}
catch (IOException ex)
{
_logger.ErrorException("Error validating IBN entry {0}", ex, currentPerson.Name);
}
// Update progress
lock (progress)
{
numComplete++;
double percent = numComplete;
percent /= people.Count;
progress.Report(100 * percent);
}
}));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
progress.Report(100);
_logger.Info("People validation complete");
}
///
/// Reloads the root media folder
///
/// The progress.
/// The cancellation token.
/// Task.
public Task ValidateMediaLibrary(IProgress progress, CancellationToken cancellationToken)
{
// Just run the scheduled task so that the user can see it
return Task.Run(() => _taskManager.CancelIfRunningAndQueue());
}
///
/// Validates the media library internal.
///
/// The progress.
/// The cancellation token.
/// Task.
public async Task ValidateMediaLibraryInternal(IProgress progress, CancellationToken cancellationToken)
{
_logger.Info("Validating media library");
await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false);
// Start by just validating the children of the root, but go no further
await RootFolder.ValidateChildren(new Progress(), cancellationToken, recursive: false);
foreach (var folder in _userManager.Users.Select(u => u.RootFolder).Distinct())
{
await ValidateCollectionFolders(folder, cancellationToken).ConfigureAwait(false);
}
// Now validate the entire media library
await RootFolder.ValidateChildren(progress, cancellationToken, recursive: true).ConfigureAwait(false);
}
///
/// Validates only the collection folders for a User and goes no further
///
/// The user root folder.
/// The cancellation token.
/// Task.
private async Task ValidateCollectionFolders(UserRootFolder userRootFolder, CancellationToken cancellationToken)
{
_logger.Info("Validating collection folders within {0}", userRootFolder.Path);
await userRootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
await userRootFolder.ValidateChildren(new Progress(), cancellationToken, recursive: false).ConfigureAwait(false);
}
///
/// Gets the default view.
///
/// IEnumerable{VirtualFolderInfo}.
public IEnumerable GetDefaultVirtualFolders()
{
return GetView(ConfigurationManager.ApplicationPaths.DefaultUserViewsPath);
}
///
/// Gets the view.
///
/// The user.
/// IEnumerable{VirtualFolderInfo}.
public IEnumerable GetVirtualFolders(User user)
{
return GetView(user.RootFolderPath);
}
///
/// Gets the view.
///
/// The path.
/// IEnumerable{VirtualFolderInfo}.
private IEnumerable GetView(string path)
{
return Directory.EnumerateDirectories(path, "*", SearchOption.TopDirectoryOnly)
.Select(dir => new VirtualFolderInfo
{
Name = Path.GetFileName(dir),
Locations = Directory.EnumerateFiles(dir, "*.lnk", SearchOption.TopDirectoryOnly).Select(FileSystem.ResolveShortcut).OrderBy(i => i).ToList()
});
}
///
/// Gets the item by id.
///
/// The id.
/// BaseItem.
/// id
public BaseItem GetItemById(Guid id)
{
if (id == Guid.Empty)
{
throw new ArgumentNullException("id");
}
BaseItem item;
LibraryItemsCache.TryGetValue(id, out item);
return item;
}
///
/// Gets the intros.
///
/// The item.
/// The user.
/// IEnumerable{System.String}.
public IEnumerable GetIntros(BaseItem item, User user)
{
return IntroProviders.SelectMany(i => i.GetIntros(item, user));
}
///
/// Sorts the specified sort by.
///
/// The items.
/// The user.
/// The sort by.
/// The sort order.
/// IEnumerable{BaseItem}.
public IEnumerable Sort(IEnumerable items, User user, IEnumerable sortBy, SortOrder sortOrder)
{
var isFirst = true;
IOrderedEnumerable orderedItems = null;
foreach (var orderBy in sortBy.Select(o => GetComparer(o, user)).Where(c => c != null))
{
if (isFirst)
{
orderedItems = sortOrder == SortOrder.Descending ? items.OrderByDescending(i => i, orderBy) : items.OrderBy(i => i, orderBy);
}
else
{
orderedItems = sortOrder == SortOrder.Descending ? orderedItems.ThenByDescending(i => i, orderBy) : orderedItems.ThenBy(i => i, orderBy);
}
isFirst = false;
}
return orderedItems ?? items;
}
///
/// Gets the comparer.
///
/// The name.
/// The user.
/// IBaseItemComparer.
private IBaseItemComparer GetComparer(string name, User user)
{
var comparer = Comparers.FirstOrDefault(c => string.Equals(name, c.Name, StringComparison.OrdinalIgnoreCase));
if (comparer != null)
{
// If it requires a user, create a new one, and assign the user
if (comparer is IUserBaseItemComparer)
{
var userComparer = (IUserBaseItemComparer)Activator.CreateInstance(comparer.GetType());
userComparer.User = user;
userComparer.UserManager = _userManager;
userComparer.UserDataRepository = _userDataRepository;
return userComparer;
}
}
return comparer;
}
///
/// Saves the item.
///
/// The item.
/// The cancellation token.
/// Task.
public Task SaveItem(BaseItem item, CancellationToken cancellationToken)
{
return ItemRepository.SaveItem(item, cancellationToken);
}
///
/// Retrieves the item.
///
/// The id.
/// Task{BaseItem}.
public BaseItem RetrieveItem(Guid id)
{
return ItemRepository.RetrieveItem(id);
}
///
/// Saves the children.
///
/// The id.
/// The children.
/// The cancellation token.
/// Task.
public Task SaveChildren(Guid id, IEnumerable children, CancellationToken cancellationToken)
{
return ItemRepository.SaveChildren(id, children, cancellationToken);
}
///
/// Retrieves the children.
///
/// The parent.
/// IEnumerable{BaseItem}.
public IEnumerable RetrieveChildren(Folder parent)
{
return ItemRepository.RetrieveChildren(parent);
}
}
}