From cc2ac9e3879619920b84aba2132c2f1323fdeb71 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 15 Jun 2016 15:52:38 -0400 Subject: [PATCH 1/3] update music brainz album responses --- .../Music/MusicBrainzAlbumProvider.cs | 160 +++++++++++++----- .../Music/MusicBrainzArtistProvider.cs | 9 +- 2 files changed, 125 insertions(+), 44 deletions(-) diff --git a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs b/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs index d76c89dfb..09a0edcf0 100644 --- a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs +++ b/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs @@ -25,7 +25,7 @@ namespace MediaBrowser.Providers.Music private readonly IApplicationHost _appHost; private readonly ILogger _logger; - public static string MusicBrainzBaseUrl = "http://musicbrainz.fercasas.com:5000"; + public static string MusicBrainzBaseUrl = "https://www.musicbrainz.org"; public MusicBrainzAlbumProvider(IHttpClient httpClient, IApplicationHost appHost, ILogger logger) { @@ -44,7 +44,7 @@ namespace MediaBrowser.Providers.Music if (!string.IsNullOrEmpty(releaseId)) { - url = string.Format(MusicBrainzBaseUrl + "/ws/2/release/?query=reid:{0}", releaseId); + url = string.Format("/ws/2/release/?query=reid:{0}", releaseId); } else { @@ -52,7 +52,7 @@ namespace MediaBrowser.Providers.Music if (!string.IsNullOrWhiteSpace(artistMusicBrainzId)) { - url = string.Format(MusicBrainzBaseUrl + "/ws/2/release/?query=\"{0}\" AND arid:{1}", + url = string.Format("/ws/2/release/?query=\"{0}\" AND arid:{1}", WebUtility.UrlEncode(searchInfo.Name), artistMusicBrainzId); } @@ -60,7 +60,7 @@ namespace MediaBrowser.Providers.Music { isNameSearch = true; - url = string.Format(MusicBrainzBaseUrl + "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", + url = string.Format("/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", WebUtility.UrlEncode(searchInfo.Name), WebUtility.UrlEncode(searchInfo.GetAlbumArtist())); } @@ -199,57 +199,86 @@ namespace MediaBrowser.Providers.Music private async Task GetReleaseResult(string albumName, string artistId, CancellationToken cancellationToken) { - var url = string.Format(MusicBrainzBaseUrl + "/ws/2/release/?query=\"{0}\" AND arid:{1}", + var url = string.Format("/ws/2/release/?query=\"{0}\" AND arid:{1}", WebUtility.UrlEncode(albumName), artistId); var doc = await GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false); - return GetReleaseResult(doc); + return ReleaseResult.Parse(doc); } private async Task GetReleaseResultByArtistName(string albumName, string artistName, CancellationToken cancellationToken) { - var url = string.Format(MusicBrainzBaseUrl + "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", + var url = string.Format("/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", WebUtility.UrlEncode(albumName), WebUtility.UrlEncode(artistName)); var doc = await GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false); - return GetReleaseResult(doc); - } - - private ReleaseResult GetReleaseResult(XmlDocument doc) - { - var ns = new XmlNamespaceManager(doc.NameTable); - ns.AddNamespace("mb", MusicBrainzBaseUrl + "/ns/mmd-2.0#"); - - var result = new ReleaseResult - { - - }; - - var releaseIdNode = doc.SelectSingleNode("//mb:release-list/mb:release/@id", ns); - - if (releaseIdNode != null) - { - result.ReleaseId = releaseIdNode.Value; - } - - var releaseGroupIdNode = doc.SelectSingleNode("//mb:release-list/mb:release/mb:release-group/@id", ns); - - if (releaseGroupIdNode != null) - { - result.ReleaseGroupId = releaseGroupIdNode.Value; - } - - return result; + return ReleaseResult.Parse(doc); } private class ReleaseResult { public string ReleaseId; public string ReleaseGroupId; + + public static ReleaseResult Parse(XmlDocument doc) + { + var docElem = doc.DocumentElement; + + if (docElem == null) + { + return new ReleaseResult(); + } + + var releaseList = docElem.FirstChild; + if (releaseList == null) + { + return new ReleaseResult(); + } + + var nodes = releaseList.ChildNodes; + string releaseId = null; + string releaseGroupId = null; + + if (nodes != null) + { + foreach (var node in nodes.Cast()) + { + if (string.Equals(node.Name, "release", StringComparison.OrdinalIgnoreCase)) + { + releaseId = node.Attributes["id"].Value; + releaseGroupId = GetReleaseGroupIdFromReleaseNode(node); + break; + } + } + } + + return new ReleaseResult + { + ReleaseId = releaseId, + ReleaseGroupId = releaseGroupId + }; + } + + private static string GetReleaseGroupIdFromReleaseNode(XmlNode node) + { + var subNodes = node.ChildNodes; + if (subNodes != null) + { + foreach (var subNode in subNodes.Cast()) + { + if (string.Equals(subNode.Name, "release-group", StringComparison.OrdinalIgnoreCase)) + { + return subNode.Attributes["id"].Value; + } + } + } + + return null; + } } /// @@ -260,7 +289,7 @@ namespace MediaBrowser.Providers.Music /// Task{System.String}. private async Task GetReleaseGroupId(string releaseEntryId, CancellationToken cancellationToken) { - var url = string.Format(MusicBrainzBaseUrl + "/ws/2/release-group/?query=reid:{0}", releaseEntryId); + var url = string.Format("/ws/2/release-group/?query=reid:{0}", releaseEntryId); var doc = await GetMusicBrainzResponse(url, false, cancellationToken).ConfigureAwait(false); @@ -276,6 +305,49 @@ namespace MediaBrowser.Providers.Music /// private readonly SemaphoreSlim _musicBrainzResourcePool = new SemaphoreSlim(1, 1); + private long _lastMbzUrlQueryTicks = 0; + private List _mbzUrls = null; + private MbzUrl _chosenUrl; + + private async Task GetMbzUrl() + { + if (_mbzUrls == null || (DateTime.UtcNow.Ticks - _lastMbzUrlQueryTicks) > TimeSpan.FromHours(12).Ticks) + { + await RefreshMzbUrls().ConfigureAwait(false); + + var urls = _mbzUrls.ToList(); + _chosenUrl = urls[new Random().Next(0, urls.Count - 1)]; + } + + return _chosenUrl; + } + + private async Task RefreshMzbUrls() + { + try + { + _mbzUrls = new List + { + new MbzUrl + { + url = MusicBrainzBaseUrl, + throttleMs = 1000 + } + }; + } + catch + { + _mbzUrls = new List + { + new MbzUrl + { + url = MusicBrainzBaseUrl, + throttleMs = 1000 + } + }; + } + } + /// /// Gets the music brainz response. /// @@ -285,9 +357,15 @@ namespace MediaBrowser.Providers.Music /// Task{XmlDocument}. internal async Task GetMusicBrainzResponse(string url, bool isSearch, CancellationToken cancellationToken) { - // MusicBrainz is extremely adamant about limiting to one request per second + var urlInfo = await GetMbzUrl().ConfigureAwait(false); - await Task.Delay(1000, cancellationToken).ConfigureAwait(false); + if (urlInfo.throttleMs > 0) + { + // MusicBrainz is extremely adamant about limiting to one request per second + await Task.Delay(urlInfo.throttleMs, cancellationToken).ConfigureAwait(false); + } + + url = urlInfo.url.TrimEnd('/') + url; var doc = new XmlDocument(); @@ -319,5 +397,11 @@ namespace MediaBrowser.Providers.Music { throw new NotImplementedException(); } + + internal class MbzUrl + { + public string url { get; set; } + public int throttleMs { get; set; } + } } } diff --git a/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs b/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs index 86563b2b9..f36062ebe 100644 --- a/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs +++ b/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs @@ -23,7 +23,7 @@ namespace MediaBrowser.Providers.Music if (!string.IsNullOrWhiteSpace(musicBrainzId)) { - var url = string.Format(MusicBrainzAlbumProvider.MusicBrainzBaseUrl + "/ws/2/artist/?query=arid:{0}", musicBrainzId); + var url = string.Format("/ws/2/artist/?query=arid:{0}", musicBrainzId); var doc = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, false, cancellationToken) .ConfigureAwait(false); @@ -35,7 +35,7 @@ namespace MediaBrowser.Providers.Music // They seem to throw bad request failures on any term with a slash var nameToSearch = searchInfo.Name.Replace('/', ' '); - var url = String.Format(MusicBrainzAlbumProvider.MusicBrainzBaseUrl + "/ws/2/artist/?query=artist:\"{0}\"", UrlEncode(nameToSearch)); + var url = String.Format("/ws/2/artist/?query=artist:\"{0}\"", UrlEncode(nameToSearch)); var doc = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false); @@ -49,7 +49,7 @@ namespace MediaBrowser.Providers.Music if (HasDiacritics(searchInfo.Name)) { // Try again using the search with accent characters url - url = String.Format(MusicBrainzAlbumProvider.MusicBrainzBaseUrl + "/ws/2/artist/?query=artistaccent:\"{0}\"", UrlEncode(nameToSearch)); + url = String.Format("/ws/2/artist/?query=artistaccent:\"{0}\"", UrlEncode(nameToSearch)); doc = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false); @@ -62,9 +62,6 @@ namespace MediaBrowser.Providers.Music private IEnumerable GetResultsFromResponse(XmlDocument doc) { - //var ns = new XmlNamespaceManager(doc.NameTable); - //ns.AddNamespace("mb", "https://musicbrainz.org/ns/mmd-2.0#"); - var list = new List(); var docElem = doc.DocumentElement; From 825f0f3507d9daa00f2caea80f834db7219675f5 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 15 Jun 2016 16:14:04 -0400 Subject: [PATCH 2/3] update music brainz album responses --- .../Music/MusicBrainzAlbumProvider.cs | 30 ++++++++++++++++--- .../Savers/MovieNfoSaver.cs | 5 ++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs b/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs index 09a0edcf0..30fe7c21a 100644 --- a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs +++ b/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs @@ -293,11 +293,33 @@ namespace MediaBrowser.Providers.Music var doc = await GetMusicBrainzResponse(url, false, cancellationToken).ConfigureAwait(false); - var ns = new XmlNamespaceManager(doc.NameTable); - ns.AddNamespace("mb", MusicBrainzBaseUrl + "/ns/mmd-2.0#"); - var node = doc.SelectSingleNode("//mb:release-group-list/mb:release-group/@id", ns); + var docElem = doc.DocumentElement; - return node != null ? node.Value : null; + if (docElem == null) + { + return null; + } + + var releaseList = docElem.FirstChild; + if (releaseList == null) + { + return null; + } + + var nodes = releaseList.ChildNodes; + string releaseGroupId = null; + + if (nodes != null) + { + foreach (var node in nodes.Cast()) + { + if (string.Equals(node.Name, "release-group", StringComparison.OrdinalIgnoreCase)) + { + return node.Attributes["id"].Value; + } + } + } + return null; } /// diff --git a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs index 18423f59e..e30a52c78 100644 --- a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs +++ b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs @@ -44,6 +44,11 @@ namespace MediaBrowser.XbmcMetadata.Savers } else { + //if (item is Movie) + //{ + // list.Add(Path.Combine(item.ContainingFolderPath, "movie.nfo")); + //} + list.Add(Path.ChangeExtension(item.Path, ".nfo")); } From 37d7db4bc459f30b9c0d415e72b320590a5328a2 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 15 Jun 2016 22:37:06 -0400 Subject: [PATCH 3/3] support xmltv gzip --- .../HttpClientManager/HttpClientManager.cs | 10 ++++--- MediaBrowser.Common/Net/HttpRequestOptions.cs | 3 ++ MediaBrowser.Controller/Providers/ItemInfo.cs | 8 ++--- .../Dto/DtoService.cs | 7 ++++- .../LiveTv/Listings/XmlTvListingsProvider.cs | 29 +++++++++++++++++-- .../Savers/MovieNfoSaver.cs | 4 ++- 6 files changed, 48 insertions(+), 13 deletions(-) diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index f9dbd766f..ce1e9fd7f 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -143,7 +143,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager }; } - private WebRequest GetRequest(HttpRequestOptions options, string method, bool enableHttpCompression) + private WebRequest GetRequest(HttpRequestOptions options, string method) { var request = CreateWebRequest(options.Url); var httpWebRequest = request as HttpWebRequest; @@ -154,7 +154,9 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager AddRequestHeaders(httpWebRequest, options); - httpWebRequest.AutomaticDecompression = enableHttpCompression ? DecompressionMethods.Deflate : DecompressionMethods.None; + httpWebRequest.AutomaticDecompression = options.EnableHttpCompression ? + (options.DecompressionMethod ?? DecompressionMethods.Deflate) : + DecompressionMethods.None; } request.CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache); @@ -366,7 +368,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager }; } - var httpWebRequest = GetRequest(options, httpMethod, options.EnableHttpCompression); + var httpWebRequest = GetRequest(options, httpMethod); if (options.RequestContentBytes != null || !string.IsNullOrEmpty(options.RequestContent) || @@ -556,7 +558,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager options.CancellationToken.ThrowIfCancellationRequested(); - var httpWebRequest = GetRequest(options, "GET", options.EnableHttpCompression); + var httpWebRequest = GetRequest(options, "GET"); if (options.ResourcePool != null) { diff --git a/MediaBrowser.Common/Net/HttpRequestOptions.cs b/MediaBrowser.Common/Net/HttpRequestOptions.cs index 75368a5fc..1a7f414a7 100644 --- a/MediaBrowser.Common/Net/HttpRequestOptions.cs +++ b/MediaBrowser.Common/Net/HttpRequestOptions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Threading; namespace MediaBrowser.Common.Net @@ -16,6 +17,8 @@ namespace MediaBrowser.Common.Net /// The URL. public string Url { get; set; } + public DecompressionMethods? DecompressionMethod { get; set; } + /// /// Gets or sets the accept header. /// diff --git a/MediaBrowser.Controller/Providers/ItemInfo.cs b/MediaBrowser.Controller/Providers/ItemInfo.cs index d16a73028..63cc48058 100644 --- a/MediaBrowser.Controller/Providers/ItemInfo.cs +++ b/MediaBrowser.Controller/Providers/ItemInfo.cs @@ -1,3 +1,4 @@ +using System; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; @@ -5,10 +6,6 @@ namespace MediaBrowser.Controller.Providers { public class ItemInfo { - public ItemInfo() - { - } - public ItemInfo(IHasMetadata item) { Path = item.Path; @@ -21,8 +18,11 @@ namespace MediaBrowser.Controller.Providers VideoType = video.VideoType; IsPlaceHolder = video.IsPlaceHolder; } + + ItemType = item.GetType(); } + public Type ItemType { get; set; } public string Path { get; set; } public string ContainingFolderPath { get; set; } public VideoType VideoType { get; set; } diff --git a/MediaBrowser.Server.Implementations/Dto/DtoService.cs b/MediaBrowser.Server.Implementations/Dto/DtoService.cs index a2d895e2c..31a35eec9 100644 --- a/MediaBrowser.Server.Implementations/Dto/DtoService.cs +++ b/MediaBrowser.Server.Implementations/Dto/DtoService.cs @@ -486,10 +486,15 @@ namespace MediaBrowser.Server.Implementations.Dto dto.UserData.Played = dto.UserData.PlayedPercentage.HasValue && dto.UserData.PlayedPercentage.Value >= 100; } - else + else if (item.SourceType == SourceType.Library) { dto.UserData = _userDataRepository.GetUserDataDto(item, user); } + else + { + var userData = _userDataRepository.GetUserData(user, item); + dto.UserData = GetUserItemDataDto(userData); + } if (item.SourceType == SourceType.Library) { diff --git a/MediaBrowser.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs b/MediaBrowser.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs index d536d3004..14e4e1093 100644 --- a/MediaBrowser.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs +++ b/MediaBrowser.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Net; +using System.Text; using System.Threading; using System.Threading.Tasks; using Emby.XmlTv.Classes; @@ -53,7 +55,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv.Listings return path; } - var cacheFilename = DateTime.UtcNow.DayOfYear.ToString(CultureInfo.InvariantCulture) + "_" + DateTime.UtcNow.Hour.ToString(CultureInfo.InvariantCulture) + ".xml"; + var cacheFilename = DateTime.UtcNow.DayOfYear.ToString(CultureInfo.InvariantCulture) + "-" + DateTime.UtcNow.Hour.ToString(CultureInfo.InvariantCulture) + ".xml"; var cacheFile = Path.Combine(_config.ApplicationPaths.CachePath, "xmltv", cacheFilename); if (File.Exists(cacheFile)) { @@ -67,13 +69,34 @@ namespace MediaBrowser.Server.Implementations.LiveTv.Listings CancellationToken = cancellationToken, Url = path, Progress = new Progress(), - EnableHttpCompression = false + DecompressionMethod = DecompressionMethods.GZip, + + // It's going to come back gzipped regardless of this value + // So we need to make sure the decompression method is set to gzip + EnableHttpCompression = true }).ConfigureAwait(false); Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)); - File.Copy(tempFile, cacheFile, true); + using (var stream = File.OpenRead(tempFile)) + { + using (var reader = new StreamReader(stream, Encoding.UTF8)) + { + using (var fileStream = File.OpenWrite(cacheFile)) + { + using (var writer = new StreamWriter(fileStream)) + { + while (!reader.EndOfStream) + { + writer.WriteLine(reader.ReadLine()); + } + } + } + } + } + + _logger.Debug("Returning xmltv path {0}", cacheFile); return cacheFile; } diff --git a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs index e30a52c78..63445b9c4 100644 --- a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs +++ b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs @@ -44,7 +44,9 @@ namespace MediaBrowser.XbmcMetadata.Savers } else { - //if (item is Movie) + // http://kodi.wiki/view/NFO_files/Movies + // movie.nfo will override all and any .nfo files in the same folder as the media files if you use the "Use foldernames for lookups" setting. If you don't, then moviename.nfo is used + //if (!item.IsInMixedFolder && item.ItemType == typeof(Movie)) //{ // list.Add(Path.Combine(item.ContainingFolderPath, "movie.nfo")); //}