reduce fanart tv requests by using their updates api

MediaBrowser.Controller/MediaBrowser.Controller.csproj

@@ -79,6 +79,7 @@
     <Compile Include="Providers\Music\ArtistsPostScanTask.cs" />
     <Compile Include="Providers\Music\FanArtUpdatesPrescanTask.cs" />
     <Compile Include="Providers\TV\FanArtSeasonProvider.cs" />
+    <Compile Include="Providers\TV\FanArtTvUpdatesPrescanTask.cs" />
     <Compile Include="Providers\TV\TvdbPrescanTask.cs" />
     <Compile Include="Providers\TV\TvdbSeriesImageProvider.cs" />
     <Compile Include="Session\ISessionManager.cs" />

MediaBrowser.Controller/Providers/Music/FanArtUpdatesPrescanTask.cs

@@ -68,7 +68,7 @@ namespace MediaBrowser.Controller.Providers.Music
                 return;
             }
 
-            // Find out the last time we queried tvdb for updates
+            // Find out the last time we queried for updates
             var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
 
             var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
@@ -93,11 +93,11 @@ namespace MediaBrowser.Controller.Providers.Music
         /// <summary>
         /// Gets the artist ids to update.
         /// </summary>
-        /// <param name="existingSeriesIds">The existing series ids.</param>
+        /// <param name="existingArtistIds">The existing series ids.</param>
         /// <param name="lastUpdateTime">The last update time.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task{IEnumerable{System.String}}.</returns>
-        private async Task<IEnumerable<string>> GetArtistIdsToUpdate(IEnumerable<string> existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken)
+        private async Task<IEnumerable<string>> GetArtistIdsToUpdate(IEnumerable<string> existingArtistIds, string lastUpdateTime, CancellationToken cancellationToken)
         {
             // First get last time
             using (var stream = await _httpClient.Get(new HttpRequestOptions
@@ -121,7 +121,7 @@ namespace MediaBrowser.Controller.Providers.Music
 
                     var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdate>>(json);
 
-                    return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase));
+                    return updates.Select(i => i.id).Where(i => existingArtistIds.Contains(i, StringComparer.OrdinalIgnoreCase));
                 }
             }
         }

MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs

@@ -10,6 +10,7 @@ using MediaBrowser.Model.Logging;
 using System;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
@@ -20,6 +21,8 @@ namespace MediaBrowser.Controller.Providers.TV
     {
         protected string FanArtBaseUrl = "http://api.fanart.tv/webservice/series/{0}/{1}/xml/all/1/1";
 
+        internal static FanArtTvProvider Current { get; private set; }
+
         /// <summary>
         /// Gets the HTTP client.
         /// </summary>
@@ -37,6 +40,7 @@ namespace MediaBrowser.Controller.Providers.TV
             }
             HttpClient = httpClient;
             _providerManager = providerManager;
+            Current = this;
         }
 
         public override bool Supports(BaseItem item)
@@ -80,7 +84,23 @@ namespace MediaBrowser.Controller.Providers.TV
         /// <returns>Guid.</returns>
         private Guid GetComparisonData(string id)
         {
-            return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5();
+            if (!string.IsNullOrEmpty(id))
+            {
+                // Process images
+                var path = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, id);
+
+                var files = new DirectoryInfo(path)
+                    .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly)
+                    .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks)
+                    .ToArray();
+
+                if (files.Length > 0)
+                {
+                    return string.Join(string.Empty, files).GetMD5();
+                }
+            }
+
+            return Guid.Empty;
         }
 
         /// <summary>
@@ -148,7 +168,6 @@ namespace MediaBrowser.Controller.Providers.TV
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            var status = ProviderRefreshStatus.Success;
             BaseProviderInfo data;
 
             if (!item.ProviderData.TryGetValue(Id, out data))
@@ -157,86 +176,95 @@ namespace MediaBrowser.Controller.Providers.TV
                 item.ProviderData[Id] = data;
             }
 
-            var series = (Series)item;
+            var seriesId = item.GetProviderId(MetadataProviders.Tvdb);
 
-            string language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
+            var seriesDataPath = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId);
+            var xmlPath = Path.Combine(seriesDataPath, "fanart.xml");
 
-            var seriesId = series.GetProviderId(MetadataProviders.Tvdb);
+            // Only download the xml if it doesn't already exist. The prescan task will take care of getting updates
-            string url = string.Format(FanArtBaseUrl, ApiKey, seriesId);
+            if (!File.Exists(xmlPath))
-
-            var xmlPath = Path.Combine(GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId), "fanart.xml");
-
-            using (var response = await HttpClient.Get(new HttpRequestOptions
             {
-                Url = url,
+                await DownloadSeriesXml(seriesDataPath, seriesId, cancellationToken).ConfigureAwait(false);
-                ResourcePool = FanArtResourcePool,
-                CancellationToken = cancellationToken
-
-            }).ConfigureAwait(false))
-            {
-                using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
-                {
-                    await response.CopyToAsync(xmlFileStream).ConfigureAwait(false);
-                }
             }
 
+            if (File.Exists(xmlPath))
+            {
+                await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false);
+            }
+
+            data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb));
+            SetLastRefreshed(item, DateTime.UtcNow);
+
+            return true;
+        }
+
+        /// <summary>
+        /// Fetches from XML.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <param name="xmlFilePath">The XML file path.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken)
+        {
             var doc = new XmlDocument();
-            doc.Load(xmlPath);
+            doc.Load(xmlFilePath);
 
             cancellationToken.ThrowIfCancellationRequested();
 
-            string path;
+            var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
+            
             var hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hdtv" : "clear";
-            if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !series.HasImage(ImageType.Logo))
+            if (ConfigurationManager.Configuration.DownloadSeriesImages.Logo && !item.HasImage(ImageType.Logo))
             {
                 var node = doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo[@lang = \"" + language + "\"]/@url") ??
                             doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo[@lang = \"" + language + "\"]/@url") ??
                             doc.SelectSingleNode("//fanart/series/" + hd + "logos/" + hd + "logo/@url") ??
                             doc.SelectSingleNode("//fanart/series/clearlogos/clearlogo/@url");
-                path = node != null ? node.Value : null;
+                var path = node != null ? node.Value : null;
                 if (!string.IsNullOrEmpty(path))
                 {
-                    series.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(series, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
+                    item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
                 }
             }
 
             cancellationToken.ThrowIfCancellationRequested();
 
             hd = ConfigurationManager.Configuration.DownloadHDFanArt ? "hd" : "";
-            if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !series.HasImage(ImageType.Art))
+            if (ConfigurationManager.Configuration.DownloadSeriesImages.Art && !item.HasImage(ImageType.Art))
             {
                 var node = doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart[@lang = \"" + language + "\"]/@url") ??
                            doc.SelectSingleNode("//fanart/series/cleararts/clearart[@lang = \"" + language + "\"]/@url") ??
                            doc.SelectSingleNode("//fanart/series/" + hd + "cleararts/" + hd + "clearart/@url") ??
                            doc.SelectSingleNode("//fanart/series/cleararts/clearart/@url");
-                path = node != null ? node.Value : null;
+                var path = node != null ? node.Value : null;
                 if (!string.IsNullOrEmpty(path))
                 {
-                    series.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(series, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
+                    item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
                 }
             }
 
             cancellationToken.ThrowIfCancellationRequested();
 
-            if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !series.HasImage(ImageType.Thumb))
+            if (ConfigurationManager.Configuration.DownloadSeriesImages.Thumb && !item.HasImage(ImageType.Thumb))
             {
                 var node = doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb[@lang = \"" + language + "\"]/@url") ??
                            doc.SelectSingleNode("//fanart/series/tvthumbs/tvthumb/@url");
-                path = node != null ? node.Value : null;
+                var path = node != null ? node.Value : null;
                 if (!string.IsNullOrEmpty(path))
                 {
-                    series.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(series, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
+                    item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
                 }
             }
 
-            if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !series.HasImage(ImageType.Banner))
+            if (ConfigurationManager.Configuration.DownloadSeriesImages.Banner && !item.HasImage(ImageType.Banner))
             {
                 var node = doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner[@lang = \"" + language + "\"]/@url") ??
                            doc.SelectSingleNode("//fanart/series/tbbanners/tvbanner/@url");
-                path = node != null ? node.Value : null;
+                var path = node != null ? node.Value : null;
                 if (!string.IsNullOrEmpty(path))
                 {
-                    series.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(series, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
+                    item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, ConfigurationManager.Configuration.SaveLocalMeta, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
                 }
             }
 
@@ -250,7 +278,7 @@ namespace MediaBrowser.Controller.Providers.TV
 
                     foreach (XmlNode node in nodes)
                     {
-                        path = node.Value;
+                        var path = node.Value;
 
                         if (!string.IsNullOrEmpty(path))
                         {
@@ -265,11 +293,37 @@ namespace MediaBrowser.Controller.Providers.TV
                 }
             }
 
-
-            data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tvdb));
-            SetLastRefreshed(series, DateTime.UtcNow, status);
-
-            return true;
         }
+
+        /// <summary>
+        /// Downloads the series XML.
+        /// </summary>
+        /// <param name="seriesDataPath">The series data path.</param>
+        /// <param name="tvdbId">The TVDB id.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        internal async Task DownloadSeriesXml(string seriesDataPath, string tvdbId, CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            string url = string.Format(FanArtBaseUrl, ApiKey, tvdbId);
+
+            var xmlPath = Path.Combine(seriesDataPath, "fanart.xml");
+
+            using (var response = await HttpClient.Get(new HttpRequestOptions
+            {
+                Url = url,
+                ResourcePool = FanArtResourcePool,
+                CancellationToken = cancellationToken
+
+            }).ConfigureAwait(false))
+            {
+                using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
+                {
+                    await response.CopyToAsync(xmlFileStream).ConfigureAwait(false);
+                }
+            }
+        }
+
     }
 }

MediaBrowser.Controller/Providers/TV/FanArtTvUpdatesPrescanTask.cs

@@ -0,0 +1,199 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Common.Net;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Providers.Music;
+using MediaBrowser.Model.Logging;
+using MediaBrowser.Model.Net;
+using MediaBrowser.Model.Serialization;
+
+namespace MediaBrowser.Controller.Providers.TV
+{
+    class FanArtTvUpdatesPrescanTask : ILibraryPrescanTask
+    {
+        private const string UpdatesUrl = "http://api.fanart.tv/webservice/newtv/{0}/{1}/";
+
+        /// <summary>
+        /// The _HTTP client
+        /// </summary>
+        private readonly IHttpClient _httpClient;
+        /// <summary>
+        /// The _logger
+        /// </summary>
+        private readonly ILogger _logger;
+        /// <summary>
+        /// The _config
+        /// </summary>
+        private readonly IServerConfigurationManager _config;
+        private readonly IJsonSerializer _jsonSerializer;
+
+        private static readonly CultureInfo UsCulture = new CultureInfo("en-US");
+
+        public FanArtTvUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient)
+        {
+            _jsonSerializer = jsonSerializer;
+            _config = config;
+            _logger = logger;
+            _httpClient = httpClient;
+        }
+
+        /// <summary>
+        /// Runs the specified progress.
+        /// </summary>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            if (!_config.Configuration.EnableInternetProviders)
+            {
+                progress.Report(100);
+                return;
+            }
+
+            var path = FanArtTvProvider.GetSeriesDataPath(_config.CommonApplicationPaths);
+
+            var timestampFile = Path.Combine(path, "time.txt");
+
+            var timestampFileInfo = new FileInfo(timestampFile);
+
+            if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays)
+            {
+                return;
+            }
+
+            // Find out the last time we queried for updates
+            var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
+
+            var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
+
+            // If this is our first time, don't do any updates and just record the timestamp
+            if (!string.IsNullOrEmpty(lastUpdateTime))
+            {
+                var seriesToUpdate = await GetSeriesIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false);
+
+                progress.Report(5);
+
+                await UpdateSeries(seriesToUpdate, path, progress, cancellationToken).ConfigureAwait(false);
+            }
+
+            var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture);
+            
+            File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8);
+
+            progress.Report(100);
+        }
+
+        /// <summary>
+        /// Gets the series ids to update.
+        /// </summary>
+        /// <param name="existingSeriesIds">The existing series ids.</param>
+        /// <param name="lastUpdateTime">The last update time.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task{IEnumerable{System.String}}.</returns>
+        private async Task<IEnumerable<string>> GetSeriesIdsToUpdate(IEnumerable<string> existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken)
+        {
+            // First get last time
+            using (var stream = await _httpClient.Get(new HttpRequestOptions
+            {
+                Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime),
+                CancellationToken = cancellationToken,
+                EnableHttpCompression = true,
+                ResourcePool = FanartBaseProvider.FanArtResourcePool
+
+            }).ConfigureAwait(false))
+            {
+                // If empty fanart will return a string of "null", rather than an empty list
+                using (var reader = new StreamReader(stream))
+                {
+                    var json = await reader.ReadToEndAsync().ConfigureAwait(false);
+
+                    if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase))
+                    {
+                        return new List<string>();
+                    }
+
+                    var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdatesPrescanTask.FanArtUpdate>>(json);
+
+                    return updates.Select(i => i.id).Where(i => existingSeriesIds.Contains(i, StringComparer.OrdinalIgnoreCase));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Updates the series.
+        /// </summary>
+        /// <param name="idList">The id list.</param>
+        /// <param name="seriesDataPath">The artists data path.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task UpdateSeries(IEnumerable<string> idList, string seriesDataPath, IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            var list = idList.ToList();
+            var numComplete = 0;
+
+            foreach (var id in list)
+            {
+                try
+                {
+                    await UpdateSeries(id, seriesDataPath, cancellationToken).ConfigureAwait(false);
+                }
+                catch (HttpException ex)
+                {
+                    // Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout
+                    if (!ex.IsTimedOut)
+                    {
+                        throw;
+                    }
+                }
+
+                numComplete++;
+                double percent = numComplete;
+                percent /= list.Count;
+                percent *= 95;
+
+                progress.Report(percent + 5);
+            }
+        }
+
+        private Task UpdateSeries(string tvdbId, string seriesDataPath, CancellationToken cancellationToken)
+        {
+            _logger.Info("Updating series " + tvdbId);
+
+            seriesDataPath = Path.Combine(seriesDataPath, tvdbId);
+
+            if (!Directory.Exists(seriesDataPath))
+            {
+                Directory.CreateDirectory(seriesDataPath);
+            }
+
+            return FanArtTvProvider.Current.DownloadSeriesXml(seriesDataPath, tvdbId, cancellationToken);
+        }
+
+        /// <summary>
+        /// Dates the time to unix timestamp.
+        /// </summary>
+        /// <param name="dateTime">The date time.</param>
+        /// <returns>System.Double.</returns>
+        private static double DateTimeToUnixTimestamp(DateTime dateTime)
+        {
+            return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds;
+        }
+
+        public class FanArtUpdate
+        {
+            public string id { get; set; }
+            public string name { get; set; }
+            public string new_images { get; set; }
+            public string total_images { get; set; }
+        }
+    }
+}