using System.Xml.Linq; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Logging; using System; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; using MediaBrowser.Providers.Extensions; namespace MediaBrowser.Providers.TV { /// /// Class RemoteSeriesProvider /// class RemoteSeriesProvider : BaseMetadataProvider, IDisposable { /// /// The _provider manager /// private readonly IProviderManager _providerManager; /// /// The tv db /// internal readonly SemaphoreSlim TvDbResourcePool = new SemaphoreSlim(2, 2); /// /// Gets the current. /// /// The current. internal static RemoteSeriesProvider Current { get; private set; } /// /// The _zip client /// private readonly IZipClient _zipClient; /// /// Gets the HTTP client. /// /// The HTTP client. protected IHttpClient HttpClient { get; private set; } /// /// Initializes a new instance of the class. /// /// The HTTP client. /// The log manager. /// The configuration manager. /// The provider manager. /// The zip client. /// httpClient public RemoteSeriesProvider(IHttpClient httpClient, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager, IZipClient zipClient) : base(logManager, configurationManager) { if (httpClient == null) { throw new ArgumentNullException("httpClient"); } HttpClient = httpClient; _providerManager = providerManager; _zipClient = zipClient; Current = this; } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool dispose) { if (dispose) { TvDbResourcePool.Dispose(); } } /// /// The root URL /// private const string RootUrl = "http://www.thetvdb.com/api/"; /// /// The series query /// private const string SeriesQuery = "GetSeries.php?seriesname={0}"; /// /// The series get zip /// private const string SeriesGetZip = "http://www.thetvdb.com/api/{0}/series/{1}/all/{2}.zip"; /// /// The LOCA l_ MET a_ FIL e_ NAME /// protected const string LocalMetaFileName = "Series.xml"; /// /// Supportses the specified item. /// /// The item. /// true if XXXX, false otherwise public override bool Supports(BaseItem item) { return item is Series; } /// /// Gets the priority. /// /// The priority. public override MetadataProviderPriority Priority { get { return MetadataProviderPriority.Second; } } /// /// Gets a value indicating whether [requires internet]. /// /// true if [requires internet]; otherwise, false. public override bool RequiresInternet { get { return true; } } /// /// Gets a value indicating whether [refresh on version change]. /// /// true if [refresh on version change]; otherwise, false. protected override bool RefreshOnVersionChange { get { return true; } } /// /// Gets the provider version. /// /// The provider version. protected override string ProviderVersion { get { return "1"; } } /// /// Needses the refresh internal. /// /// The item. /// The provider info. /// true if XXXX, false otherwise protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) { // Refresh even if local metadata exists because we need episode infos if (GetComparisonData(item) != providerInfo.Data) { return true; } return base.NeedsRefreshInternal(item, providerInfo); } /// /// Gets the comparison data. /// /// The item. /// Guid. private Guid GetComparisonData(BaseItem item) { var seriesId = item.GetProviderId(MetadataProviders.Tvdb); if (!string.IsNullOrEmpty(seriesId)) { // Process images var path = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId); var files = new DirectoryInfo(path) .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly) .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks) .ToArray(); if (files.Length > 0) { return string.Join(string.Empty, files).GetMD5(); } } return Guid.Empty; } /// /// Fetches metadata and returns true or false indicating if any work that requires persistence was done /// /// The item. /// if set to true [force]. /// The cancellation token. /// Task{System.Boolean}. public override async Task FetchAsync(BaseItem item, bool force, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var series = (Series)item; var seriesId = series.GetProviderId(MetadataProviders.Tvdb); if (string.IsNullOrEmpty(seriesId)) { seriesId = await GetSeriesId(series, cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); if (!string.IsNullOrEmpty(seriesId)) { series.SetProviderId(MetadataProviders.Tvdb, seriesId); var seriesDataPath = GetSeriesDataPath(ConfigurationManager.ApplicationPaths, seriesId); await FetchSeriesData(series, seriesId, seriesDataPath, cancellationToken).ConfigureAwait(false); } BaseProviderInfo data; if (!item.ProviderData.TryGetValue(Id, out data)) { data = new BaseProviderInfo(); item.ProviderData[Id] = data; } data.Data = GetComparisonData(item); SetLastRefreshed(item, DateTime.UtcNow); return true; } /// /// Fetches the series data. /// /// The series. /// The series id. /// The series data path. /// The cancellation token. /// Task{System.Boolean}. private async Task FetchSeriesData(Series series, string seriesId, string seriesDataPath, CancellationToken cancellationToken) { var files = Directory.EnumerateFiles(seriesDataPath, "*.xml", SearchOption.TopDirectoryOnly).Select(Path.GetFileName).ToArray(); var seriesXmlFilename = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower() + ".xml"; // Only download if not already there // The prescan task will take care of updates so we don't need to re-download here if (!files.Contains("banners.xml", StringComparer.OrdinalIgnoreCase) || !files.Contains("actors.xml", StringComparer.OrdinalIgnoreCase) || !files.Contains(seriesXmlFilename, StringComparer.OrdinalIgnoreCase)) { await DownloadSeriesZip(seriesId, seriesDataPath, cancellationToken).ConfigureAwait(false); } // Examine if there's no local metadata, or save local is on (to get updates) if (!HasLocalMeta(series) || ConfigurationManager.Configuration.SaveLocalMeta) { var seriesXmlPath = Path.Combine(seriesDataPath, seriesXmlFilename); var actorsXmlPath = Path.Combine(seriesDataPath, "actors.xml"); var seriesDoc = new XmlDocument(); seriesDoc.Load(seriesXmlPath); FetchMainInfo(series, seriesDoc); var actorsDoc = new XmlDocument(); actorsDoc.Load(actorsXmlPath); FetchActors(series, actorsDoc, seriesDoc); if (ConfigurationManager.Configuration.SaveLocalMeta) { var ms = new MemoryStream(); seriesDoc.Save(ms); await _providerManager.SaveToLibraryFilesystem(series, Path.Combine(series.MetaLocation, LocalMetaFileName), ms, cancellationToken).ConfigureAwait(false); } } } /// /// Downloads the series zip. /// /// The series id. /// The series data path. /// The cancellation token. /// Task. internal async Task DownloadSeriesZip(string seriesId, string seriesDataPath, CancellationToken cancellationToken) { var url = string.Format(SeriesGetZip, TVUtils.TvdbApiKey, seriesId, ConfigurationManager.Configuration.PreferredMetadataLanguage); using (var zipStream = await HttpClient.Get(new HttpRequestOptions { Url = url, ResourcePool = TvDbResourcePool, CancellationToken = cancellationToken }).ConfigureAwait(false)) { // Copy to memory stream because we need a seekable stream using (var ms = new MemoryStream()) { await zipStream.CopyToAsync(ms).ConfigureAwait(false); ms.Position = 0; _zipClient.ExtractAll(ms, seriesDataPath, true); } } } /// /// Gets the series data path. /// /// The app paths. /// The series id. /// System.String. internal static string GetSeriesDataPath(IApplicationPaths appPaths, string seriesId) { var seriesDataPath = Path.Combine(GetSeriesDataPath(appPaths), seriesId); if (!Directory.Exists(seriesDataPath)) { Directory.CreateDirectory(seriesDataPath); } return seriesDataPath; } /// /// Gets the series data path. /// /// The app paths. /// System.String. internal static string GetSeriesDataPath(IApplicationPaths appPaths) { var dataPath = Path.Combine(appPaths.DataPath, "tvdb"); if (!Directory.Exists(dataPath)) { Directory.CreateDirectory(dataPath); } return dataPath; } /// /// Fetches the main info. /// /// The series. /// The doc. private void FetchMainInfo(Series series, XmlDocument doc) { series.Name = doc.SafeGetString("//SeriesName"); series.Overview = doc.SafeGetString("//Overview"); series.CommunityRating = doc.SafeGetSingle("//Rating", 0, 10); series.AirDays = TVUtils.GetAirDays(doc.SafeGetString("//Airs_DayOfWeek")); series.AirTime = doc.SafeGetString("//Airs_Time"); SeriesStatus seriesStatus; if(Enum.TryParse(doc.SafeGetString("//Status"), true, out seriesStatus)) series.Status = seriesStatus; series.PremiereDate = doc.SafeGetDateTime("//FirstAired"); if (series.PremiereDate.HasValue) series.ProductionYear = series.PremiereDate.Value.Year; //Runtime is in minutes, and 1 tick = 10000 ms series.RunTimeTicks = doc.SafeGetInt32("//Runtime") * 6; string s = doc.SafeGetString("//Network"); if (!string.IsNullOrWhiteSpace(s)) { series.Studios.Clear(); foreach (var studio in s.Trim().Split('|')) { series.AddStudio(studio); } } series.OfficialRating = doc.SafeGetString("//ContentRating"); string g = doc.SafeGetString("//Genre"); if (g != null) { string[] genres = g.Trim('|').Split('|'); if (g.Length > 0) { series.Genres.Clear(); foreach (var genre in genres) { series.AddGenre(genre); } } } if (series.Status == SeriesStatus.Ended) { var document = XDocument.Load(new XmlNodeReader(doc)); var dates = document.Descendants("Episode").Where(x => { var seasonNumber = x.Element("SeasonNumber"); var firstAired = x.Element("FirstAired"); return firstAired != null && seasonNumber != null && (!string.IsNullOrEmpty(seasonNumber.Value) && seasonNumber.Value != "0") && !string.IsNullOrEmpty(firstAired.Value); }).Select(x => { DateTime? date = null; DateTime tempDate; var firstAired = x.Element("FirstAired"); if (firstAired != null && DateTime.TryParse(firstAired.Value, out tempDate)) { date = tempDate; } return date; }).ToList(); if(dates.Any(x=>x.HasValue)) series.EndDate = dates.Where(x => x.HasValue).Max(); } } /// /// Fetches the actors. /// /// The series. /// The actors doc. /// The seriesDoc. /// Task. private void FetchActors(Series series, XmlDocument actorsDoc, XmlDocument seriesDoc) { XmlNode actorsNode = null; if (ConfigurationManager.Configuration.SaveLocalMeta) { //add to the main seriesDoc for saving var seriesNode = seriesDoc.SelectSingleNode("//Series"); if (seriesNode != null) { actorsNode = seriesDoc.CreateNode(XmlNodeType.Element, "Persons", null); seriesNode.AppendChild(actorsNode); } } var xmlNodeList = actorsDoc.SelectNodes("Actors/Actor"); if (xmlNodeList != null) { series.People.Clear(); foreach (XmlNode p in xmlNodeList) { string actorName = p.SafeGetString("Name"); string actorRole = p.SafeGetString("Role"); if (!string.IsNullOrWhiteSpace(actorName)) { series.AddPerson(new PersonInfo { Type = PersonType.Actor, Name = actorName.Trim(), Role = actorRole }); if (ConfigurationManager.Configuration.SaveLocalMeta && actorsNode != null) { //create in main seriesDoc var personNode = seriesDoc.CreateNode(XmlNodeType.Element, "Person", null); foreach (XmlNode subNode in p.ChildNodes) personNode.AppendChild(seriesDoc.ImportNode(subNode, true)); //need to add the type var typeNode = seriesDoc.CreateNode(XmlNodeType.Element, "Type", null); typeNode.InnerText = PersonType.Actor; personNode.AppendChild(typeNode); actorsNode.AppendChild(personNode); } } } } } /// /// The us culture /// protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// /// Determines whether [has local meta] [the specified item]. /// /// The item. /// true if [has local meta] [the specified item]; otherwise, false. private bool HasLocalMeta(BaseItem item) { return item.ResolveArgs.ContainsMetaFileByName(LocalMetaFileName); } /// /// Gets the series id. /// /// The item. /// The cancellation token. /// Task{System.String}. private async Task GetSeriesId(BaseItem item, CancellationToken cancellationToken) { var seriesId = item.GetProviderId(MetadataProviders.Tvdb); if (string.IsNullOrEmpty(seriesId)) { seriesId = await FindSeries(item.Name, cancellationToken).ConfigureAwait(false); } return seriesId; } /// /// Finds the series. /// /// The name. /// The cancellation token. /// Task{System.String}. public async Task FindSeries(string name, CancellationToken cancellationToken) { //nope - search for it string url = string.Format(RootUrl + SeriesQuery, WebUtility.UrlEncode(name)); var doc = new XmlDocument(); using (var results = await HttpClient.Get(new HttpRequestOptions { Url = url, ResourcePool = TvDbResourcePool, CancellationToken = cancellationToken }).ConfigureAwait(false)) { doc.Load(results); } if (doc.HasChildNodes) { XmlNodeList nodes = doc.SelectNodes("//Series"); string comparableName = GetComparableName(name); if (nodes != null) foreach (XmlNode node in nodes) { var n = node.SelectSingleNode("./SeriesName"); if (n != null && GetComparableName(n.InnerText) == comparableName) { n = node.SelectSingleNode("./seriesid"); if (n != null) return n.InnerText; } else { if (n != null) Logger.Info("TVDb Provider - " + n.InnerText + " did not match " + comparableName); } } } Logger.Info("TVDb Provider - Could not find " + name + ". Check name on Thetvdb.org."); return null; } /// /// The remove /// const string remove = "\"'!`?"; /// /// The spacers /// const string spacers = "/,.:;\\(){}[]+-_=–*"; // (there are not actually two - in the they are different char codes) /// /// Gets the name of the comparable. /// /// The name. /// System.String. internal static string GetComparableName(string name) { name = name.ToLower(); name = name.Normalize(NormalizationForm.FormKD); var sb = new StringBuilder(); foreach (var c in name) { if ((int)c >= 0x2B0 && (int)c <= 0x0333) { // skip char modifier and diacritics } else if (remove.IndexOf(c) > -1) { // skip chars we are removing } else if (spacers.IndexOf(c) > -1) { sb.Append(" "); } else if (c == '&') { sb.Append(" and "); } else { sb.Append(c); } } name = sb.ToString(); name = name.Replace(", the", ""); name = name.Replace("the ", " "); name = name.Replace(" the ", " "); string prevName; do { prevName = name; name = name.Replace(" ", " "); } while (name.Length != prevName.Length); return name.Trim(); } /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// public void Dispose() { Dispose(true); } } }