diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index d3f8ba927..d5eac2cee 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -79,6 +79,7 @@ + diff --git a/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs b/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs index bedab16c1..82040c9d6 100644 --- a/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs +++ b/MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs @@ -68,7 +68,7 @@ namespace MediaBrowser.Controller.Providers.Music return; } - // Find out the last time we queried tvdb for updates + // Find out the last time we queried for updates var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); @@ -93,11 +93,11 @@ namespace MediaBrowser.Controller.Providers.Music /// /// Gets the artist ids to update. /// - /// The existing series ids. + /// The existing series ids. /// The last update time. /// The cancellation token. /// Task{IEnumerable{System.String}}. - private async Task> GetArtistIdsToUpdate(IEnumerable existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken) + private async Task> GetArtistIdsToUpdate(IEnumerable existingArtistIds, string lastUpdateTime, CancellationToken cancellationToken) { // First get last time using (var stream = await _httpClient.Get(new HttpRequestOptions @@ -121,7 +121,7 @@ namespace MediaBrowser.Controller.Providers.Music var updates = _jsonSerializer.DeserializeFromString>(json); - return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase)); + return updates.Select(i => i.id).Where(i => existingArtistIds.Contains(i, StringComparer.OrdinalIgnoreCase)); } } } diff --git a/MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs b/MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs index e5e66e942..64b69a817 100644 --- a/MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs +++ b/MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs @@ -10,6 +10,7 @@ using MediaBrowser.Model.Logging; using System; using System.Globalization; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -20,6 +21,8 @@ namespace MediaBrowser.Controller.Providers.TV { protected string FanArtBaseUrl = "http://api.fanart.tv/webservice/series/{0}/{1}/xml/all/1/1"; + internal static FanArtTvProvider Current { get; private set; } + /// /// Gets the HTTP client. /// @@ -37,6 +40,7 @@ namespace MediaBrowser.Controller.Providers.TV } HttpClient = httpClient; _providerManager = providerManager; + Current = this; } public override bool Supports(BaseItem item) @@ -80,7 +84,23 @@ namespace MediaBrowser.Controller.Providers.TV /// Guid. private Guid GetComparisonData(string id) { - return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5(); + if (!string.IsNullOrEmpty(id)) + { + // Process images + var path = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, id); + + var files = new DirectoryInfo(path) + .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly) + .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks) + .ToArray(); + + if (files.Length > 0) + { + return string.Join(string.Empty, files).GetMD5(); + } + } + + return Guid.Empty; } /// @@ -148,7 +168,6 @@ namespace MediaBrowser.Controller.Providers.TV { cancellationToken.ThrowIfCancellationRequested(); - var status = ProviderRefreshStatus.Success; BaseProviderInfo data; if (!item.ProviderData.TryGetValue(Id, out data)) @@ -157,86 +176,95 @@ namespace MediaBrowser.Controller.Providers.TV item.ProviderData[Id] = data; } - var series = (Series)item; + var seriesId = item.GetProviderId(MetadataProviders.Tvdb); - string language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); + var seriesDataPath = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId); + var xmlPath = Path.Combine(seriesDataPath, "fanart.xml"); - var seriesId = series.GetProviderId(MetadataProviders.Tvdb); - string url = string.Format(FanArtBaseUrl, ApiKey, seriesId); - - var xmlPath = Path.Combine(GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId), "fanart.xml"); - - using (var response = await HttpClient.Get(new HttpRequestOptions + // Only download the xml if it doesn't already exist. The prescan task will take care of getting updates + if (!File.Exists(xmlPath)) { - Url = url, - ResourcePool = FanArtResourcePool, - CancellationToken = cancellationToken - - }).ConfigureAwait(false)) - { - using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) - { - await response.CopyToAsync(xmlFileStream).ConfigureAwait(false); - } + await DownloadSeriesXml(seriesDataPath, seriesId, cancellationToken).ConfigureAwait(false); } + if (File.Exists(xmlPath)) + { + await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false); + } + + data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb)); + SetLastRefreshed(item, DateTime.UtcNow); + + return true; + } + + /// + /// Fetches from XML. + /// + /// The item. + /// The XML file path. + /// The cancellation token. + /// Task. + private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken) + { var doc = new XmlDocument(); - doc.Load(xmlPath); + doc.Load(xmlFilePath); cancellationToken.ThrowIfCancellationRequested(); - string path; + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); + var hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hdtv" : "clear"; - if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !series.HasImage(ImageType.Logo)) + if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !item.HasImage(ImageType.Logo)) { var node = doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo/@url") ?? doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo/@url"); - path = node != null ? node.Value : null; + var path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - series.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(series, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hd" : ""; - if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !series.HasImage(ImageType.Art)) + if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !item.HasImage(ImageType.Art)) { var node = doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/cleararts/clearart[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart/@url") ?? doc.SelectSingleNode("//fanart/series/cleararts/clearart/@url"); - path = node != null ? node.Value : null; + var path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - series.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(series, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); - if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !series.HasImage(ImageType.Thumb)) + if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !item.HasImage(ImageType.Thumb)) { var node = doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb/@url"); - path = node != null ? node.Value : null; + var path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - series.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(series, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } - if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !series.HasImage(ImageType.Banner)) + if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !item.HasImage(ImageType.Banner)) { var node = doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner[@lang = \"" + language + "\"]/@url") ?? doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner/@url"); - path = node != null ? node.Value : null; + var path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - series.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(series, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -250,7 +278,7 @@ namespace MediaBrowser.Controller.Providers.TV foreach (XmlNode node in nodes) { - path = node.Value; + var path = node.Value; if (!string.IsNullOrEmpty(path)) { @@ -265,11 +293,37 @@ namespace MediaBrowser.Controller.Providers.TV } } - - data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb)); - SetLastRefreshed(series, DateTime.UtcNow, status); - - return true; } + + /// + /// Downloads the series XML. + /// + /// The series data path. + /// The TVDB id. + /// The cancellation token. + /// Task. + internal async Task DownloadSeriesXml(string seriesDataPath, string tvdbId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + string url = string.Format(FanArtBaseUrl, ApiKey, tvdbId); + + var xmlPath = Path.Combine(seriesDataPath, "fanart.xml"); + + using (var response = await HttpClient.Get(new HttpRequestOptions + { + Url = url, + ResourcePool = FanArtResourcePool, + CancellationToken = cancellationToken + + }).ConfigureAwait(false)) + { + using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await response.CopyToAsync(xmlFileStream).ConfigureAwait(false); + } + } + } + } } diff --git a/MediaBrowser.Controller/Providers/TV/FanArtTvUpdatesPrescanTask.cs b/MediaBrowser.Controller/Providers/TV/FanArtTvUpdatesPrescanTask.cs new file mode 100644 index 000000000..d2c70f5c8 --- /dev/null +++ b/MediaBrowser.Controller/Providers/TV/FanArtTvUpdatesPrescanTask.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers.Music; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; + +namespace MediaBrowser.Controller.Providers.TV +{ + class FanArtTvUpdatesPrescanTask : ILibraryPrescanTask + { + private const string UpdatesUrl = "http://api.fanart.tv/webservice/newtv/{0}/{1}/"; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + /// + /// The _logger + /// + private readonly ILogger _logger; + /// + /// The _config + /// + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _jsonSerializer; + + private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + public FanArtTvUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient) + { + _jsonSerializer = jsonSerializer; + _config = config; + _logger = logger; + _httpClient = httpClient; + } + + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + if (!_config.Configuration.EnableInternetProviders) + { + progress.Report(100); + return; + } + + var path = FanArtTvProvider.GetSeriesDataPath(_config.CommonApplicationPaths); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays) + { + return; + } + + // Find out the last time we queried for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + // If this is our first time, don't do any updates and just record the timestamp + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + var seriesToUpdate = await GetSeriesIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false); + + progress.Report(5); + + await UpdateSeries(seriesToUpdate, path, progress, cancellationToken).ConfigureAwait(false); + } + + var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture); + + File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8); + + progress.Report(100); + } + + /// + /// Gets the series ids to update. + /// + /// The existing series ids. + /// The last update time. + /// The cancellation token. + /// Task{IEnumerable{System.String}}. + private async Task> GetSeriesIdsToUpdate(IEnumerable existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken) + { + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = FanartBaseProvider.FanArtResourcePool + + }).ConfigureAwait(false)) + { + // If empty fanart will return a string of "null", rather than an empty list + using (var reader = new StreamReader(stream)) + { + var json = await reader.ReadToEndAsync().ConfigureAwait(false); + + if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase)) + { + return new List(); + } + + var updates = _jsonSerializer.DeserializeFromString>(json); + + return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase)); + } + } + } + + /// + /// Updates the series. + /// + /// The id list. + /// The artists data path. + /// The progress. + /// The cancellation token. + /// Task. + private async Task UpdateSeries(IEnumerable idList, string seriesDataPath, IProgress progress, CancellationToken cancellationToken) + { + var list = idList.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdateSeries(id, seriesDataPath, cancellationToken).ConfigureAwait(false); + } + catch (HttpException ex) + { + // Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout + if (!ex.IsTimedOut) + { + throw; + } + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 95; + + progress.Report(percent + 5); + } + } + + private Task UpdateSeries(string tvdbId, string seriesDataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating series " + tvdbId); + + seriesDataPath = Path.Combine(seriesDataPath, tvdbId); + + if (!Directory.Exists(seriesDataPath)) + { + Directory.CreateDirectory(seriesDataPath); + } + + return FanArtTvProvider.Current.DownloadSeriesXml(seriesDataPath, tvdbId, cancellationToken); + } + + /// + /// Dates the time to unix timestamp. + /// + /// The date time. + /// System.Double. + private static double DateTimeToUnixTimestamp(DateTime dateTime) + { + return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds; + } + + public class FanArtUpdate + { + public string id { get; set; } + public string name { get; set; } + public string new_images { get; set; } + public string total_images { get; set; } + } + } +}