diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index d5eac2cee..1a97201d8 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -75,6 +75,7 @@ + diff --git a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs index 75aee1a04..dc71a860c 100644 --- a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -7,6 +9,8 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; using System.Globalization; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -16,7 +20,7 @@ namespace MediaBrowser.Controller.Providers.Movies /// /// Class FanArtMovieProvider /// - class FanArtMovieProvider : FanartBaseProvider, IDisposable + class FanArtMovieProvider : FanartBaseProvider { /// /// Gets the HTTP client. @@ -24,10 +28,18 @@ namespace MediaBrowser.Controller.Providers.Movies /// The HTTP client. protected IHttpClient HttpClient { get; private set; } + /// + /// The _provider manager + /// private readonly IProviderManager _providerManager; + /// + /// The us culture + /// private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + internal static FanArtMovieProvider Current { get; private set; } + /// /// Initializes a new instance of the class. /// @@ -45,18 +57,7 @@ namespace MediaBrowser.Controller.Providers.Movies } HttpClient = httpClient; _providerManager = providerManager; - } - - /// - /// Releases unmanaged and - optionally - managed resources. - /// - /// true to release both managed and unmanaged resources; false to release only unmanaged resources. - protected virtual void Dispose(bool dispose) - { - if (dispose) - { - FanArtResourcePool.Dispose(); - } + Current = this; } /// @@ -79,7 +80,7 @@ namespace MediaBrowser.Controller.Providers.Movies { get { - return "12"; + return "13"; } } @@ -140,10 +141,62 @@ namespace MediaBrowser.Controller.Providers.Movies /// /// Gets the comparison data. /// + /// The id. /// Guid. private Guid GetComparisonData(string id) { - return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5(); + if (!string.IsNullOrEmpty(id)) + { + // Process images + var path = GetMovieDataPath(ConfigurationManager.ApplicationPaths, id); + + var files = new DirectoryInfo(path) + .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly) + .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks) + .ToArray(); + + if (files.Length > 0) + { + return string.Join(string.Empty, files).GetMD5(); + } + } + + return Guid.Empty; + } + + /// + /// Gets the movie data path. + /// + /// The app paths. + /// The TMDB id. + /// System.String. + internal static string GetMovieDataPath(IApplicationPaths appPaths, string tmdbId) + { + var dataPath = Path.Combine(GetMoviesDataPath(appPaths), tmdbId); + + if (!Directory.Exists(dataPath)) + { + Directory.CreateDirectory(dataPath); + } + + return dataPath; + } + + /// + /// Gets the movie data path. + /// + /// The app paths. + /// System.String. + internal static string GetMoviesDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.DataPath, "fanart-movies"); + + if (!Directory.Exists(dataPath)) + { + Directory.CreateDirectory(dataPath); + } + + return dataPath; } /// @@ -165,15 +218,43 @@ namespace MediaBrowser.Controller.Providers.Movies item.ProviderData[Id] = data; } - var status = ProviderRefreshStatus.Success; + var movieId = item.GetProviderId(MetadataProviders.Tmdb); - var movie = item; + var movieDataPath = GetMovieDataPath(ConfigurationManager.ApplicationPaths, movieId); + var xmlPath = Path.Combine(movieDataPath, "fanart.xml"); - var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); - var url = string.Format(FanArtBaseUrl, ApiKey, movie.GetProviderId(MetadataProviders.Tmdb)); - var doc = new XmlDocument(); + // Only download the xml if it doesn't already exist. The prescan task will take care of getting updates + if (!File.Exists(xmlPath)) + { + await DownloadMovieXml(movieDataPath, movieId, cancellationToken).ConfigureAwait(false); + } - using (var xml = await HttpClient.Get(new HttpRequestOptions + if (File.Exists(xmlPath)) + { + await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false); + } + + data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb)); + SetLastRefreshed(item, DateTime.UtcNow); + return true; + } + + /// + /// Downloads the movie XML. + /// + /// The movie data path. + /// The TMDB id. + /// The cancellation token. + /// Task. + internal async Task DownloadMovieXml(string movieDataPath, string tmdbId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + string url = string.Format(FanArtBaseUrl, ApiKey, tmdbId); + + var xmlPath = Path.Combine(movieDataPath, "fanart.xml"); + + using (var response = await HttpClient.Get(new HttpRequestOptions { Url = url, ResourcePool = FanArtResourcePool, @@ -181,9 +262,27 @@ namespace MediaBrowser.Controller.Providers.Movies }).ConfigureAwait(false)) { - doc.Load(xml); + using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await response.CopyToAsync(xmlFileStream).ConfigureAwait(false); + } } + } + /// + /// Fetches from XML. + /// + /// The item. + /// The XML file path. + /// The cancellation token. + /// Task. + private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken) + { + var doc = new XmlDocument(); + doc.Load(xmlFilePath); + + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); + cancellationToken.ThrowIfCancellationRequested(); var saveLocal = ConfigurationManager.Configuration.SaveLocalMeta && @@ -205,7 +304,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(movie, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); @@ -220,7 +319,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(movie, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); @@ -232,7 +331,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(movie, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(item, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -245,7 +344,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(movie, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -258,7 +357,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(movie, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -277,7 +376,7 @@ namespace MediaBrowser.Controller.Providers.Movies if (!string.IsNullOrEmpty(path)) { item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, path, ("backdrop" + (numBackdrops > 0 ? numBackdrops.ToString(UsCulture) : "") + ".jpg"), saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - + numBackdrops++; if (item.BackdropImagePaths.Count >= ConfigurationManager.Configuration.MaxBackdrops) break; @@ -286,15 +385,6 @@ namespace MediaBrowser.Controller.Providers.Movies } } - - data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb)); - SetLastRefreshed(movie, DateTime.UtcNow, status); - return true; - } - - public void Dispose() - { - Dispose(true); } } } diff --git a/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs b/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs new file mode 100644 index 000000000..727271f9c --- /dev/null +++ b/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs @@ -0,0 +1,184 @@ +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers.Music; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Providers.Movies +{ + class FanArtMovieUpdatesPrescanTask : ILibraryPrescanTask + { + private const string UpdatesUrl = "http://api.fanart.tv/webservice/newmovies/{0}/{1}/"; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + /// + /// The _logger + /// + private readonly ILogger _logger; + /// + /// The _config + /// + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _jsonSerializer; + + private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + public FanArtMovieUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient) + { + _jsonSerializer = jsonSerializer; + _config = config; + _logger = logger; + _httpClient = httpClient; + } + + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + if (!_config.Configuration.EnableInternetProviders) + { + progress.Report(100); + return; + } + + var path = FanArtMovieProvider.GetMoviesDataPath(_config.CommonApplicationPaths); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays) + { + return; + } + + // Find out the last time we queried for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + // If this is our first time, don't do any updates and just record the timestamp + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + var moviesToUpdate = await GetMovieIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false); + + progress.Report(5); + + await UpdateMovies(moviesToUpdate, path, progress, cancellationToken).ConfigureAwait(false); + } + + var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture); + + File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8); + + progress.Report(100); + } + + private async Task> GetMovieIdsToUpdate(IEnumerable existingIds, string lastUpdateTime, CancellationToken cancellationToken) + { + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = FanartBaseProvider.FanArtResourcePool + + }).ConfigureAwait(false)) + { + // If empty fanart will return a string of "null", rather than an empty list + using (var reader = new StreamReader(stream)) + { + var json = await reader.ReadToEndAsync().ConfigureAwait(false); + + if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase)) + { + return new List(); + } + + var updates = _jsonSerializer.DeserializeFromString>(json); + + return updates.Select(i => i.id).Where(i => existingIds.Contains(i, StringComparer.OrdinalIgnoreCase)); + } + } + } + + private async Task UpdateMovies(IEnumerable idList, string moviesDataPath, IProgress progress, CancellationToken cancellationToken) + { + var list = idList.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdateMovie(id, moviesDataPath, cancellationToken).ConfigureAwait(false); + } + catch (HttpException ex) + { + // Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout + if (!ex.IsTimedOut) + { + throw; + } + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 95; + + progress.Report(percent + 5); + } + } + + private Task UpdateMovie(string tmdbId, string movieDataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating movie " + tmdbId); + + movieDataPath = Path.Combine(movieDataPath, tmdbId); + + if (!Directory.Exists(movieDataPath)) + { + Directory.CreateDirectory(movieDataPath); + } + + return FanArtMovieProvider.Current.DownloadMovieXml(movieDataPath, tmdbId, cancellationToken); + } + + /// + /// Dates the time to unix timestamp. + /// + /// The date time. + /// System.Double. + private static double DateTimeToUnixTimestamp(DateTime dateTime) + { + return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds; + } + + public class FanArtUpdate + { + public string id { get; set; } + public string name { get; set; } + public string new_images { get; set; } + public string total_images { get; set; } + } + } +} diff --git a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs index ced342a9c..dd9d8353c 100644 --- a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs @@ -3,7 +3,6 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; -using MediaBrowser.Model.Net; using MediaBrowser.Model.Serialization; using System; using System.Collections.Generic; diff --git a/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs b/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs index 893715b73..023df3177 100644 --- a/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs +++ b/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs @@ -358,7 +358,7 @@ namespace MediaBrowser.Server.Implementations.HttpServer { var address = endpoint.ToString(); - _localEndPoints.AddOrUpdate(address, address, (key, existing) => address); + _localEndPoints.GetOrAdd(address, address); } if (EnableHttpRequestLogging)