From 99528878b76f498f991873d57ee990f55266244d Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 02:23:44 +0200 Subject: [PATCH 1/9] OmdbProvider: Added local caching The lack of caching in the OmdbProvider could result in long library scan time. This commit adds caching for the OmdbProvider similar to the MovieDbProvider. Downloaded metadata is saved locally and only updated if the last refresh occured 3 or more days before --- .../Omdb/OmdbItemProvider.cs | 14 +++- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 79 ++++++++++++++++--- .../TV/Omdb/OmdbEpisodeProvider.cs | 14 +++- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs b/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs index a0d60c166..7a803eb6f 100644 --- a/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.Entities.TV; @@ -26,13 +28,17 @@ namespace MediaBrowser.Providers.Omdb private readonly IHttpClient _httpClient; private readonly ILogger _logger; private readonly ILibraryManager _libraryManager; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; - public OmdbItemProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager) + public OmdbItemProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; _logger = logger; _libraryManager = libraryManager; + _fileSystem = fileSystem; + _configurationManager = configurationManager; } public Task> GetSearchResults(SeriesInfo searchInfo, CancellationToken cancellationToken) @@ -220,7 +226,7 @@ namespace MediaBrowser.Providers.Omdb result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; @@ -259,7 +265,7 @@ namespace MediaBrowser.Providers.Omdb result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index 39da70210..d3c895b9a 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -1,4 +1,7 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Serialization; @@ -17,15 +20,19 @@ namespace MediaBrowser.Providers.Omdb { internal static readonly SemaphoreSlim ResourcePool = new SemaphoreSlim(1, 1); private readonly IJsonSerializer _jsonSerializer; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; private readonly IHttpClient _httpClient; private readonly CultureInfo _usCulture = new CultureInfo("en-US"); public static OmdbProvider Current; - public OmdbProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient) + public OmdbProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; + _fileSystem = fileSystem; + _configurationManager = configurationManager; Current = this; } @@ -37,24 +44,17 @@ namespace MediaBrowser.Providers.Omdb throw new ArgumentNullException("imdbId"); } - var imdbParam = imdbId.StartsWith("tt", StringComparison.OrdinalIgnoreCase) ? imdbId : "tt" + imdbId; + var path = await EnsureItemInfo(imdbId, cancellationToken); - var url = string.Format("https://www.omdbapi.com/?i={0}&tomatoes=true", imdbParam); + string resultString; - using (var stream = await _httpClient.Get(new HttpRequestOptions + using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) { - Url = url, - ResourcePool = ResourcePool, - CancellationToken = cancellationToken - - }).ConfigureAwait(false)) - { - string resultString; - using (var reader = new StreamReader(stream, new UTF8Encoding(false))) { resultString = reader.ReadToEnd(); } + } resultString = resultString.Replace("\"N/A\"", "\"\""); @@ -130,7 +130,60 @@ namespace MediaBrowser.Providers.Omdb } ParseAdditionalMetadata(item, result); + } + + internal async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(imdbId)) + { + throw new ArgumentNullException("imdbId"); } + + var imdbParam = imdbId.StartsWith("tt", StringComparison.OrdinalIgnoreCase) ? imdbId : "tt" + imdbId; + + var path = GetDataFilePath(imdbParam); + + var fileInfo = _fileSystem.GetFileSystemInfo(path); + + if (fileInfo.Exists) + { + // If it's recent or automatic updates are enabled, don't re-download + if ((DateTime.UtcNow - _fileSystem.GetLastWriteTimeUtc(fileInfo)).TotalDays <= 3) + { + return path; + } + } + + var url = string.Format("https://www.omdbapi.com/?i={0}&tomatoes=true", imdbParam); + + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = url, + ResourcePool = ResourcePool, + CancellationToken = cancellationToken + + }).ConfigureAwait(false)) + { + var rootObject = _jsonSerializer.DeserializeFromStream(stream); + _fileSystem.CreateDirectory(Path.GetDirectoryName(path)); + _jsonSerializer.SerializeToFile(rootObject, path); + } + + return path; + } + + internal string GetDataFilePath(string imdbId) + { + if (string.IsNullOrEmpty(imdbId)) + { + throw new ArgumentNullException("imdbId"); + } + + var dataPath = Path.Combine(_configurationManager.ApplicationPaths.CachePath, "omdb"); + + var filename = string.Format("{0}.json", imdbId); + + return Path.Combine(dataPath, filename); } private void ParseAdditionalMetadata(BaseItem item, RootObject result) diff --git a/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs b/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs index 5da1fcf27..ebbefbeb1 100644 --- a/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs +++ b/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -21,12 +23,16 @@ namespace MediaBrowser.Providers.TV private readonly IJsonSerializer _jsonSerializer; private readonly IHttpClient _httpClient; private readonly OmdbItemProvider _itemProvider; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; - public OmdbEpisodeProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager) + public OmdbEpisodeProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; - _itemProvider = new OmdbItemProvider(jsonSerializer, httpClient, logger, libraryManager); + _fileSystem = fileSystem; + _configurationManager = configurationManager; + _itemProvider = new OmdbItemProvider(jsonSerializer, httpClient, logger, libraryManager, fileSystem, configurationManager); } public Task> GetSearchResults(EpisodeInfo searchInfo, CancellationToken cancellationToken) @@ -58,7 +64,7 @@ namespace MediaBrowser.Providers.TV result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; From f601614539559ba943bce0bbd0138d39768b2038 Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 02:35:35 +0200 Subject: [PATCH 2/9] minifix --- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index d3c895b9a..823a29d54 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -53,11 +53,10 @@ namespace MediaBrowser.Providers.Omdb using (var reader = new StreamReader(stream, new UTF8Encoding(false))) { resultString = reader.ReadToEnd(); + resultString = resultString.Replace("\"N/A\"", "\"\""); } } - resultString = resultString.Replace("\"N/A\"", "\"\""); - var result = _jsonSerializer.DeserializeFromString(resultString); // Only take the name and rating if the user's language is set to english, since Omdb has no localization From f7900e77570f87a4a55ff11a087e8a98ae1f5d5d Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 03:08:27 +0200 Subject: [PATCH 3/9] OmdbImageProvider: Check poster field of cached item to determine if an image is available --- .../Omdb/OmdbImageProvider.cs | 35 ++++++++--------- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 38 +++++++++++-------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs index a1e038374..e795d638d 100644 --- a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs @@ -7,6 +7,8 @@ using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Providers; using System.Collections.Generic; +using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -29,22 +31,27 @@ namespace MediaBrowser.Providers.Omdb }; } - public Task> GetImages(IHasImages item, CancellationToken cancellationToken) + public async Task> GetImages(IHasImages item, CancellationToken cancellationToken) { var imdbId = item.GetProviderId(MetadataProviders.Imdb); var list = new List(); - if (!string.IsNullOrWhiteSpace(imdbId)) + if (!string.IsNullOrWhiteSpace(imdbId) && OmdbProvider.Current != null) { - list.Add(new RemoteImageInfo + OmdbProvider.RootObject rootObject = await OmdbProvider.Current.GetRootObject(imdbId, cancellationToken); + + if (!string.IsNullOrEmpty(rootObject.Poster)) { - ProviderName = Name, - Url = string.Format("https://img.omdbapi.com/?i={0}&apikey=82e83907", imdbId) - }); + list.Add(new RemoteImageInfo + { + ProviderName = Name, + Url = string.Format("https://img.omdbapi.com/?i={0}&apikey=82e83907", imdbId) + }); + } } - return Task.FromResult>(list); + return list; } public Task GetImageResponse(string url, CancellationToken cancellationToken) @@ -65,18 +72,6 @@ namespace MediaBrowser.Providers.Omdb public bool Supports(IHasImages item) { - // We'll hammer Omdb if we enable this - if (item is Person) - { - return false; - } - - // Save the http requests since we know it's not currently supported - if (item is Season || item is Episode) - { - return false; - } - // Supports images for tv movies var tvProgram = item as LiveTvProgram; if (tvProgram != null && tvProgram.IsMovie) @@ -84,7 +79,7 @@ namespace MediaBrowser.Providers.Omdb return true; } - return item is Movie || item is Trailer; + return item is Movie || item is Trailer || item is Episode; } public int Order diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index 823a29d54..9873447be 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -44,20 +44,7 @@ namespace MediaBrowser.Providers.Omdb throw new ArgumentNullException("imdbId"); } - var path = await EnsureItemInfo(imdbId, cancellationToken); - - string resultString; - - using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) - { - using (var reader = new StreamReader(stream, new UTF8Encoding(false))) - { - resultString = reader.ReadToEnd(); - resultString = resultString.Replace("\"N/A\"", "\"\""); - } - } - - var result = _jsonSerializer.DeserializeFromString(resultString); + var result = await GetRootObject(imdbId, cancellationToken); // Only take the name and rating if the user's language is set to english, since Omdb has no localization if (string.Equals(language, "en", StringComparison.OrdinalIgnoreCase)) @@ -131,7 +118,26 @@ namespace MediaBrowser.Providers.Omdb ParseAdditionalMetadata(item, result); } - internal async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) + internal async Task GetRootObject(string imdbId, CancellationToken cancellationToken) + { + var path = await EnsureItemInfo(imdbId, cancellationToken); + + string resultString; + + using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) + { + using (var reader = new StreamReader(stream, new UTF8Encoding(false))) + { + resultString = reader.ReadToEnd(); + resultString = resultString.Replace("\"N/A\"", "\"\""); + } + } + + var result = _jsonSerializer.DeserializeFromString(resultString); + return result; + } + + private async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(imdbId)) { @@ -236,7 +242,7 @@ namespace MediaBrowser.Providers.Omdb return string.Equals(lang, "en", StringComparison.OrdinalIgnoreCase); } - private class RootObject + internal class RootObject { public string Title { get; set; } public string Year { get; set; } From 7e22ce5f5e145f29c2eafb7596163822797144d5 Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 04:56:04 +0200 Subject: [PATCH 4/9] OmdbProvider instantiation --- .../Omdb/OmdbImageProvider.cs | 19 +++++++++++++++---- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 4 ---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs index e795d638d..563118940 100644 --- a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.Entities.TV; @@ -6,6 +8,7 @@ using MediaBrowser.Controller.LiveTv; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Providers; +using MediaBrowser.Model.Serialization; using System.Collections.Generic; using System.IO; using System.Text; @@ -17,10 +20,16 @@ namespace MediaBrowser.Providers.Omdb public class OmdbImageProvider : IRemoteImageProvider, IHasOrder { private readonly IHttpClient _httpClient; + private readonly IJsonSerializer _jsonSerializer; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; - public OmdbImageProvider(IHttpClient httpClient) + public OmdbImageProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { + _jsonSerializer = jsonSerializer; _httpClient = httpClient; + _fileSystem = fileSystem; + _configurationManager = configurationManager; } public IEnumerable GetSupportedImages(IHasImages item) @@ -37,9 +46,11 @@ namespace MediaBrowser.Providers.Omdb var list = new List(); - if (!string.IsNullOrWhiteSpace(imdbId) && OmdbProvider.Current != null) + var provider = new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager); + + if (!string.IsNullOrWhiteSpace(imdbId)) { - OmdbProvider.RootObject rootObject = await OmdbProvider.Current.GetRootObject(imdbId, cancellationToken); + OmdbProvider.RootObject rootObject = await provider.GetRootObject(imdbId, cancellationToken).ConfigureAwait(false); if (!string.IsNullOrEmpty(rootObject.Poster)) { diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index 9873447be..4056073f2 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -25,16 +25,12 @@ namespace MediaBrowser.Providers.Omdb private readonly IHttpClient _httpClient; private readonly CultureInfo _usCulture = new CultureInfo("en-US"); - public static OmdbProvider Current; - public OmdbProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; _fileSystem = fileSystem; _configurationManager = configurationManager; - - Current = this; } public async Task Fetch(BaseItem item, string imdbId, string language, string country, CancellationToken cancellationToken) From 79128e6d8519c731ccb49f9d982d9eec1d211eeb Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 04:47:30 +0200 Subject: [PATCH 5/9] MovieDbSeasonProvider: Don't fill name from search info --- MediaBrowser.Providers/TV/TheMovieDb/MovieDbSeasonProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbSeasonProvider.cs b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbSeasonProvider.cs index 2e51393e3..194af5b99 100644 --- a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbSeasonProvider.cs +++ b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbSeasonProvider.cs @@ -58,7 +58,7 @@ namespace MediaBrowser.Providers.TV result.HasMetadata = true; result.Item = new Season(); - result.Item.Name = info.Name; + result.Item.Name = seasonInfo.name; result.Item.IndexNumber = seasonNumber; result.Item.Overview = seasonInfo.overview; From 75840c7065e8b16fc5391811a34a06990d592388 Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 06:30:06 +0200 Subject: [PATCH 6/9] MovieDbSeriesProvider and MovieDbEpisodeProvider: Add support for trailers --- .../TV/TheMovieDb/MovieDbEpisodeProvider.cs | 16 +++++++++ .../TV/TheMovieDb/MovieDbProviderBase.cs | 14 +++++++- .../TV/TheMovieDb/MovieDbSeriesProvider.cs | 33 +++++++++++++++++-- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbEpisodeProvider.cs b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbEpisodeProvider.cs index 9bab3d380..bc9842b73 100644 --- a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbEpisodeProvider.cs +++ b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbEpisodeProvider.cs @@ -114,6 +114,22 @@ namespace MediaBrowser.Providers.TV item.CommunityRating = (float)response.vote_average; item.VoteCount = response.vote_count; + if (response.videos != null && response.videos.results != null) + { + foreach (var video in response.videos.results) + { + if (video.type.Equals("trailer", System.StringComparison.OrdinalIgnoreCase) + || video.type.Equals("clip", System.StringComparison.OrdinalIgnoreCase)) + { + if (video.site.Equals("youtube", System.StringComparison.OrdinalIgnoreCase)) + { + var videoUrl = string.Format("http://www.youtube.com/watch?v={0}", video.key); + item.AddTrailerUrl(videoUrl, true); + } + } + } + } + result.ResetPeople(); var credits = response.credits; diff --git a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbProviderBase.cs b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbProviderBase.cs index 36800202f..a6df245b0 100644 --- a/MediaBrowser.Providers/TV/TheMovieDb/MovieDbProviderBase.cs +++ b/MediaBrowser.Providers/TV/TheMovieDb/MovieDbProviderBase.cs @@ -210,7 +210,19 @@ namespace MediaBrowser.Providers.TV public class Videos { - public List results { get; set; } + public List