From 5c5ec6e644921b8f31f0fce74aadb59bf0fef99a Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Sat, 25 May 2013 11:16:50 -0400 Subject: [PATCH] further reduce fanart music requests by using their updates api --- .../MediaBrowser.Controller.csproj | 1 + .../Providers/FanartBaseProvider.cs | 7 +- .../Providers/ImagesByNameProvider.cs | 16 +- .../Providers/Movies/FanArtMovieProvider.cs | 2 - .../Providers/Music/FanArtArtistProvider.cs | 271 ++++++++++++------ .../Music/FanArtUpdatesPrescanTask.cs | 199 +++++++++++++ .../Providers/TV/RemoteSeriesProvider.cs | 4 +- .../Providers/TV/TvdbPrescanTask.cs | 22 +- .../Library/LibraryManager.cs | 68 ++++- 9 files changed, 478 insertions(+), 112 deletions(-) create mode 100644 MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 236c8b520..19e6c1747 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -74,6 +74,7 @@ + diff --git a/MediaBrowser.Controller/Providers/FanartBaseProvider.cs b/MediaBrowser.Controller/Providers/FanartBaseProvider.cs index c1249adee..f5dff25f6 100644 --- a/MediaBrowser.Controller/Providers/FanartBaseProvider.cs +++ b/MediaBrowser.Controller/Providers/FanartBaseProvider.cs @@ -10,7 +10,7 @@ namespace MediaBrowser.Controller.Providers /// public abstract class FanartBaseProvider : BaseMetadataProvider { - protected static readonly SemaphoreSlim FanArtResourcePool = new SemaphoreSlim(3,3); + internal static readonly SemaphoreSlim FanArtResourcePool = new SemaphoreSlim(3, 3); /// /// The LOG o_ FILE @@ -50,9 +50,10 @@ namespace MediaBrowser.Controller.Providers /// /// The API key /// - protected const string ApiKey = "5c6b04c68e904cfed1e6cbc9a9e683d4"; + internal const string ApiKey = "5c6b04c68e904cfed1e6cbc9a9e683d4"; - protected FanartBaseProvider(ILogManager logManager, IServerConfigurationManager configurationManager) : base(logManager, configurationManager) + protected FanartBaseProvider(ILogManager logManager, IServerConfigurationManager configurationManager) + : base(logManager, configurationManager) { } diff --git a/MediaBrowser.Controller/Providers/ImagesByNameProvider.cs b/MediaBrowser.Controller/Providers/ImagesByNameProvider.cs index 3611607c9..60c313d21 100644 --- a/MediaBrowser.Controller/Providers/ImagesByNameProvider.cs +++ b/MediaBrowser.Controller/Providers/ImagesByNameProvider.cs @@ -153,14 +153,20 @@ namespace MediaBrowser.Controller.Providers { var location = GetLocation(item); - var result = new FileInfo(Path.Combine(location, filenameWithoutExtension + ".png")); + var files = new DirectoryInfo(location).EnumerateFiles("*", SearchOption.TopDirectoryOnly).ToList(); - if (!result.Exists) - result = new FileInfo(Path.Combine(location, filenameWithoutExtension + ".jpg")); + var file = files.FirstOrDefault(i => string.Equals(i.Name, filenameWithoutExtension + ".png", StringComparison.OrdinalIgnoreCase)); - if (result.Exists) + if (file != null) { - return result; + return file; + } + + file = files.FirstOrDefault(i => string.Equals(i.Name, filenameWithoutExtension + ".jpg", StringComparison.OrdinalIgnoreCase)); + + if (file != null) + { + return file; } return null; diff --git a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs index 00d6c72ca..75aee1a04 100644 --- a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs @@ -5,9 +5,7 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; -using MediaBrowser.Model.Net; using System; -using System.Collections.Generic; using System.Globalization; using System.Threading; using System.Threading.Tasks; diff --git a/MediaBrowser.Controller/Providers/Music/FanArtArtistProvider.cs b/MediaBrowser.Controller/Providers/Music/FanArtArtistProvider.cs index 22075a08a..7f4ac7c22 100644 --- a/MediaBrowser.Controller/Providers/Music/FanArtArtistProvider.cs +++ b/MediaBrowser.Controller/Providers/Music/FanArtArtistProvider.cs @@ -1,5 +1,4 @@ -using System.IO; -using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; @@ -11,6 +10,8 @@ using MediaBrowser.Model.Logging; using System; using System.Collections.Generic; using System.Globalization; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -28,8 +29,21 @@ namespace MediaBrowser.Controller.Providers.Music /// The HTTP client. protected IHttpClient HttpClient { get; private set; } + /// + /// The _provider manager + /// private readonly IProviderManager _providerManager; + internal static FanArtArtistProvider Current; + + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client. + /// The log manager. + /// The configuration manager. + /// The provider manager. + /// httpClient public FanArtArtistProvider(IHttpClient httpClient, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) : base(logManager, configurationManager) { @@ -39,6 +53,8 @@ namespace MediaBrowser.Controller.Providers.Music } HttpClient = httpClient; _providerManager = providerManager; + + Current = this; } /// @@ -56,11 +72,19 @@ namespace MediaBrowser.Controller.Providers.Music return item is MusicArtist; } + /// + /// Gets a value indicating whether [save local meta]. + /// + /// true if [save local meta]; otherwise, false. protected virtual bool SaveLocalMeta { get { return ConfigurationManager.Configuration.SaveLocalMeta; } } + /// + /// Gets a value indicating whether [refresh on version change]. + /// + /// true if [refresh on version change]; otherwise, false. protected override bool RefreshOnVersionChange { get @@ -69,11 +93,15 @@ namespace MediaBrowser.Controller.Providers.Music } } + /// + /// Gets the provider version. + /// + /// The provider version. protected override string ProviderVersion { get { - return "5"; + return "7"; } } @@ -91,17 +119,57 @@ namespace MediaBrowser.Controller.Providers.Music } if (!ConfigurationManager.Configuration.DownloadMusicArtistImages.Art && - !ConfigurationManager.Configuration.DownloadMusicArtistImages.Backdrops && - !ConfigurationManager.Configuration.DownloadMusicArtistImages.Banner && - !ConfigurationManager.Configuration.DownloadMusicArtistImages.Logo && - !ConfigurationManager.Configuration.DownloadMusicArtistImages.Primary) + !ConfigurationManager.Configuration.DownloadMusicArtistImages.Backdrops && + !ConfigurationManager.Configuration.DownloadMusicArtistImages.Banner && + !ConfigurationManager.Configuration.DownloadMusicArtistImages.Logo && + !ConfigurationManager.Configuration.DownloadMusicArtistImages.Primary && + + // The fanart album provider depends on xml downloaded here, so honor it's settings too + !ConfigurationManager.Configuration.DownloadMusicAlbumImages.Disc && + !ConfigurationManager.Configuration.DownloadMusicAlbumImages.Primary) { return false; } + if (GetComparisonData(item) != providerInfo.Data) + { + return true; + } + return base.NeedsRefreshInternal(item, providerInfo); } + /// + /// Gets the comparison data. + /// + /// The item. + /// Guid. + private Guid GetComparisonData(BaseItem item) + { + var musicBrainzId = item.GetProviderId(MetadataProviders.Musicbrainz); + + if (!string.IsNullOrEmpty(musicBrainzId)) + { + // Process images + var path = GetArtistDataPath(ConfigurationManager.ApplicationPaths, musicBrainzId); + + var files = new DirectoryInfo(path) + .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly) + .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks) + .ToArray(); + + if (files.Length > 0) + { + return string.Join(string.Empty, files).GetMD5(); + } + } + + return Guid.Empty; + } + + /// + /// The us culture + /// protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// @@ -138,7 +206,7 @@ namespace MediaBrowser.Controller.Providers.Music return dataPath; } - + /// /// Fetches metadata and returns true or false indicating if any work that requires persistence was done /// @@ -150,15 +218,57 @@ namespace MediaBrowser.Controller.Providers.Music { cancellationToken.ThrowIfCancellationRequested(); - //var artist = item; - var musicBrainzId = item.GetProviderId(MetadataProviders.Musicbrainz); + + var artistDataPath = GetArtistDataPath(ConfigurationManager.ApplicationPaths, musicBrainzId); + var xmlPath = Path.Combine(artistDataPath, "fanart.xml"); + + // Only download the xml if it doesn't already exist. The prescan task will take care of getting updates + if (!File.Exists(xmlPath)) + { + await DownloadArtistXml(artistDataPath, musicBrainzId, cancellationToken).ConfigureAwait(false); + } + + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Art || + ConfigurationManager.Configuration.DownloadMusicArtistImages.Backdrops || + ConfigurationManager.Configuration.DownloadMusicArtistImages.Banner || + ConfigurationManager.Configuration.DownloadMusicArtistImages.Logo || + ConfigurationManager.Configuration.DownloadMusicArtistImages.Primary) + { + if (File.Exists(xmlPath)) + { + await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false); + } + } + + BaseProviderInfo data; + if (!item.ProviderData.TryGetValue(Id, out data)) + { + data = new BaseProviderInfo(); + item.ProviderData[Id] = data; + } + + data.Data = GetComparisonData(item); + + SetLastRefreshed(item, DateTime.UtcNow); + return true; + } + + /// + /// Downloads the artist XML. + /// + /// The artist path. + /// The music brainz id. + /// The cancellation token. + /// Task{System.Boolean}. + internal async Task DownloadArtistXml(string artistPath, string musicBrainzId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var url = string.Format(FanArtBaseUrl, ApiKey, musicBrainzId); - var status = ProviderRefreshStatus.Success; + var xmlPath = Path.Combine(artistPath, "fanart.xml"); - var xmlPath = Path.Combine(GetArtistDataPath(ConfigurationManager.ApplicationPaths, musicBrainzId), "fanart.xml"); - using (var response = await HttpClient.Get(new HttpRequestOptions { Url = url, @@ -172,98 +282,97 @@ namespace MediaBrowser.Controller.Providers.Music await response.CopyToAsync(xmlFileStream).ConfigureAwait(false); } } - + } + + /// + /// Fetches from XML. + /// + /// The item. + /// The XML file path. + /// The cancellation token. + /// Task. + private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken) + { var doc = new XmlDocument(); - doc.Load(xmlPath); + doc.Load(xmlFilePath); cancellationToken.ThrowIfCancellationRequested(); - if (doc.HasChildNodes) + string path; + var hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hd" : ""; + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Logo && !item.HasImage(ImageType.Logo)) { - string path; - var hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hd" : ""; - if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Logo && !item.HasImage(ImageType.Logo)) + var node = + doc.SelectSingleNode("//fanart/music/musiclogos/" + hd + "musiclogo/@url") ?? + doc.SelectSingleNode("//fanart/music/musiclogos/musiclogo/@url"); + path = node != null ? node.Value : null; + if (!string.IsNullOrEmpty(path)) { - var node = - doc.SelectSingleNode("//fanart/music/musiclogos/" + hd + "musiclogo/@url") ?? - doc.SelectSingleNode("//fanart/music/musiclogos/musiclogo/@url"); - path = node != null ? node.Value : null; - if (!string.IsNullOrEmpty(path)) - { - Logger.Debug("FanArtProvider getting ClearLogo for " + item.Name); - item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - } + item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } - cancellationToken.ThrowIfCancellationRequested(); + } + cancellationToken.ThrowIfCancellationRequested(); - if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Backdrops && item.BackdropImagePaths.Count == 0) + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Backdrops && item.BackdropImagePaths.Count == 0) + { + var nodes = doc.SelectNodes("//fanart/music/artistbackgrounds//@url"); + if (nodes != null) { - var nodes = doc.SelectNodes("//fanart/music/artistbackgrounds//@url"); - if (nodes != null) + var numBackdrops = 0; + item.BackdropImagePaths = new List(); + foreach (XmlNode node in nodes) { - var numBackdrops = 0; - item.BackdropImagePaths = new List(); - foreach (XmlNode node in nodes) + path = node.Value; + if (!string.IsNullOrEmpty(path)) { - path = node.Value; - if (!string.IsNullOrEmpty(path)) - { - Logger.Debug("FanArtProvider getting Backdrop for " + item.Name); - item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, path, ("Backdrop" + (numBackdrops > 0 ? numBackdrops.ToString(UsCulture) : "") + ".jpg"), SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - numBackdrops++; - if (numBackdrops >= ConfigurationManager.Configuration.MaxBackdrops) break; - } + item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, path, ("Backdrop" + (numBackdrops > 0 ? numBackdrops.ToString(UsCulture) : "") + ".jpg"), SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + numBackdrops++; + if (numBackdrops >= ConfigurationManager.Configuration.MaxBackdrops) break; } - } } - cancellationToken.ThrowIfCancellationRequested(); + } - if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Art && !item.HasImage(ImageType.Art)) + cancellationToken.ThrowIfCancellationRequested(); + + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Art && !item.HasImage(ImageType.Art)) + { + var node = + doc.SelectSingleNode("//fanart/music/musicarts/" + hd + "musicart/@url") ?? + doc.SelectSingleNode("//fanart/music/musicarts/musicart/@url"); + path = node != null ? node.Value : null; + if (!string.IsNullOrEmpty(path)) { - var node = - doc.SelectSingleNode("//fanart/music/musicarts/" + hd + "musicart/@url") ?? - doc.SelectSingleNode("//fanart/music/musicarts/musicart/@url"); - path = node != null ? node.Value : null; - if (!string.IsNullOrEmpty(path)) - { - Logger.Debug("FanArtProvider getting ClearArt for " + item.Name); - item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - } + item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } - cancellationToken.ThrowIfCancellationRequested(); + } + cancellationToken.ThrowIfCancellationRequested(); - if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Banner && !item.HasImage(ImageType.Banner)) + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Banner && !item.HasImage(ImageType.Banner)) + { + var node = doc.SelectSingleNode("//fanart/music/musicbanners/" + hd + "musicbanner/@url") ?? + doc.SelectSingleNode("//fanart/music/musicbanners/musicbanner/@url"); + path = node != null ? node.Value : null; + if (!string.IsNullOrEmpty(path)) { - var node = doc.SelectSingleNode("//fanart/music/musicbanners/" + hd + "musicbanner/@url") ?? - doc.SelectSingleNode("//fanart/music/musicbanners/musicbanner/@url"); - path = node != null ? node.Value : null; - if (!string.IsNullOrEmpty(path)) - { - Logger.Debug("FanArtProvider getting Banner for " + item.Name); - item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - } - } - - cancellationToken.ThrowIfCancellationRequested(); - - // Artist thumbs are actually primary images (they are square/portrait) - if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Primary && !item.HasImage(ImageType.Primary)) - { - var node = doc.SelectSingleNode("//fanart/music/artistthumbs/artistthumb/@url"); - path = node != null ? node.Value : null; - if (!string.IsNullOrEmpty(path)) - { - Logger.Debug("FanArtProvider getting Primary image for " + item.Name); - item.SetImage(ImageType.Primary, await _providerManager.DownloadAndSaveImage(item, path, PrimaryFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - } + item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } - SetLastRefreshed(item, DateTime.UtcNow, status); - return true; + cancellationToken.ThrowIfCancellationRequested(); + + // Artist thumbs are actually primary images (they are square/portrait) + if (ConfigurationManager.Configuration.DownloadMusicArtistImages.Primary && !item.HasImage(ImageType.Primary)) + { + var node = doc.SelectSingleNode("//fanart/music/artistthumbs/artistthumb/@url"); + path = node != null ? node.Value : null; + if (!string.IsNullOrEmpty(path)) + { + item.SetImage(ImageType.Primary, await _providerManager.DownloadAndSaveImage(item, path, PrimaryFile, SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + } + } } } } diff --git a/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs b/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs new file mode 100644 index 000000000..c4766a23d --- /dev/null +++ b/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs @@ -0,0 +1,199 @@ +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Providers.Music +{ + class FanArtUpdatesPrescanTask : ILibraryPrescanTask + { + private const string UpdatesUrl = "http://api.fanart.tv/webservice/newmusic/{0}/{1}/"; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + /// + /// The _logger + /// + private readonly ILogger _logger; + /// + /// The _config + /// + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _jsonSerializer; + + private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + public FanArtUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient) + { + _jsonSerializer = jsonSerializer; + _config = config; + _logger = logger; + _httpClient = httpClient; + } + + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + var path = FanArtArtistProvider.GetArtistDataPath(_config.CommonApplicationPaths); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays) + { + return; + } + + // Find out the last time we queried tvdb for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + // If this is our first time, don't do any updates and just record the timestamp + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + var artistsToUpdate = await GetArtistIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false); + + progress.Report(5); + + await UpdateArtists(artistsToUpdate, path, progress, cancellationToken).ConfigureAwait(false); + } + + var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture); + + File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8); + + progress.Report(100); + } + + /// + /// Gets the artist ids to update. + /// + /// The existing series ids. + /// The last update time. + /// The cancellation token. + /// Task{IEnumerable{System.String}}. + private async Task> GetArtistIdsToUpdate(IEnumerable existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken) + { + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = FanartBaseProvider.FanArtResourcePool + + }).ConfigureAwait(false)) + { + // If empty fanart will return a string of "null", rather than an empty list + using (var reader = new StreamReader(stream)) + { + var json = await reader.ReadToEndAsync().ConfigureAwait(false); + + if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase)) + { + return new List(); + } + + var updates = _jsonSerializer.DeserializeFromString>(json); + + return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase)); + } + } + } + + /// + /// Updates the artists. + /// + /// The id list. + /// The artists data path. + /// The progress. + /// The cancellation token. + /// Task. + private async Task UpdateArtists(IEnumerable idList, string artistsDataPath, IProgress progress, CancellationToken cancellationToken) + { + var list = idList.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdateArtist(id, artistsDataPath, cancellationToken).ConfigureAwait(false); + } + catch (HttpException ex) + { + // Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout + if (!ex.IsTimedOut) + { + throw; + } + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 95; + + progress.Report(percent + 5); + } + } + + /// + /// Updates the artist. + /// + /// The musicBrainzId. + /// The artists data path. + /// The cancellation token. + /// Task. + private Task UpdateArtist(string musicBrainzId, string artistsDataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating artist " + musicBrainzId); + + artistsDataPath = Path.Combine(artistsDataPath, musicBrainzId); + + if (!Directory.Exists(artistsDataPath)) + { + Directory.CreateDirectory(artistsDataPath); + } + + return FanArtArtistProvider.Current.DownloadArtistXml(artistsDataPath, musicBrainzId, cancellationToken); + } + + /// + /// Dates the time to unix timestamp. + /// + /// The date time. + /// System.Double. + private static double DateTimeToUnixTimestamp(DateTime dateTime) + { + return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds; + } + + public class FanArtUpdate + { + public string id { get; set; } + public string name { get; set; } + public string new_images { get; set; } + public string total_images { get; set; } + } + } +} diff --git a/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs b/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs index cf3bcbd70..3dbdea7bf 100644 --- a/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs +++ b/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs @@ -185,8 +185,7 @@ namespace MediaBrowser.Controller.Providers.TV /// Guid. private Guid GetComparisonData(BaseItem item) { - var series = (Series)item; - var seriesId = series.GetProviderId(MetadataProviders.Tvdb); + var seriesId = item.GetProviderId(MetadataProviders.Tvdb); if (!string.IsNullOrEmpty(seriesId)) { @@ -206,6 +205,7 @@ namespace MediaBrowser.Controller.Providers.TV return Guid.Empty; } + /// /// Fetches metadata and returns true or false indicating if any work that requires persistence was done /// diff --git a/MediaBrowser.Controller/Providers/TV/TvdbPrescanTask.cs b/MediaBrowser.Controller/Providers/TV/TvdbPrescanTask.cs index 3a2f47a5e..4dee31017 100644 --- a/MediaBrowser.Controller/Providers/TV/TvdbPrescanTask.cs +++ b/MediaBrowser.Controller/Providers/TV/TvdbPrescanTask.cs @@ -103,7 +103,7 @@ namespace MediaBrowser.Controller.Providers.TV newUpdateTime = doc.SafeGetString("//Time"); } - await UpdateSeries(existingDirectories, path, cancellationToken).ConfigureAwait(false); + await UpdateSeries(existingDirectories, path, progress, cancellationToken).ConfigureAwait(false); } else { @@ -111,10 +111,11 @@ namespace MediaBrowser.Controller.Providers.TV newUpdateTime = seriesToUpdate.Item2; - await UpdateSeries(seriesToUpdate.Item1, path, cancellationToken).ConfigureAwait(false); + await UpdateSeries(seriesToUpdate.Item1, path, progress, cancellationToken).ConfigureAwait(false); } File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8); + progress.Report(100); } /// @@ -158,11 +159,15 @@ namespace MediaBrowser.Controller.Providers.TV /// /// The series ids. /// The series data path. + /// The progress. /// The cancellation token. /// Task. - private async Task UpdateSeries(IEnumerable seriesIds, string seriesDataPath, CancellationToken cancellationToken) + private async Task UpdateSeries(IEnumerable seriesIds, string seriesDataPath, IProgress progress, CancellationToken cancellationToken) { - foreach (var seriesId in seriesIds) + var list = seriesIds.ToList(); + var numComplete = 0; + + foreach (var seriesId in list) { try { @@ -171,12 +176,19 @@ namespace MediaBrowser.Controller.Providers.TV catch (HttpException ex) { // Already logged at lower levels, but don't fail the whole operation, unless timed out - + // We have to fail this to make it run again otherwise new episode data could potentially be missing if (ex.IsTimedOut) { throw; } } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 100; + + progress.Report(percent); } } diff --git a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs index af0212738..ae0607689 100644 --- a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs +++ b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs @@ -252,7 +252,7 @@ namespace MediaBrowser.Server.Implementations.Library var newSeasonZeroName = ConfigurationManager.Configuration.SeasonZeroDisplayName; var seasonZeroNameChanged = !string.Equals(_seasonZeroDisplayName, newSeasonZeroName, StringComparison.CurrentCulture); - + RecordConfigurationValues(config); Task.Run(async () => @@ -895,30 +895,28 @@ namespace MediaBrowser.Server.Implementations.Library await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); + progress.Report(.5); + // Start by just validating the children of the root, but go no further await RootFolder.ValidateChildren(new Progress(), cancellationToken, recursive: false); + progress.Report(1); + foreach (var folder in _userManager.Users.Select(u => u.RootFolder).Distinct()) { await ValidateCollectionFolders(folder, cancellationToken).ConfigureAwait(false); } - // Run prescan tasks - foreach (var task in PrescanTasks) - { - try - { - await task.Run(new Progress(), cancellationToken); - } - catch (Exception ex) - { - _logger.ErrorException("Error running prescan task", ex); - } - } + progress.Report(2); + // Run prescan tasks + await RunPrescanTasks(progress, cancellationToken).ConfigureAwait(false); + + progress.Report(15); + var innerProgress = new ActionableProgress(); - innerProgress.RegisterAction(pct => progress.Report(pct * .8)); + innerProgress.RegisterAction(pct => progress.Report(15 + pct * .65)); // Now validate the entire media library await RootFolder.ValidateChildren(innerProgress, cancellationToken, recursive: true).ConfigureAwait(false); @@ -932,6 +930,48 @@ namespace MediaBrowser.Server.Implementations.Library progress.Report(100); } + /// + /// Runs the prescan tasks. + /// + /// The progress. + /// The cancellation token. + /// Task. + private async Task RunPrescanTasks(IProgress progress, CancellationToken cancellationToken) + { + var prescanTasks = PrescanTasks.ToList(); + var progressDictionary = new Dictionary(); + + var tasks = prescanTasks.Select(i => Task.Run(async () => + { + var innerProgress = new ActionableProgress(); + + innerProgress.RegisterAction(pct => + { + lock (progressDictionary) + { + progressDictionary[i] = pct; + + double percent = progressDictionary.Values.Sum(); + percent /= prescanTasks.Count; + + progress.Report(2 + percent * .13); + } + }); + + try + { + await i.Run(innerProgress, cancellationToken); + } + catch (Exception ex) + { + _logger.ErrorException("Error running prescan task", ex); + } + })); + + // Run prescan tasks + await Task.WhenAll(tasks).ConfigureAwait(false); + } + /// /// Validates only the collection folders for a User and goes no further ///