jellyfin-server/MediaBrowser.Providers/TV/MissingEpisodeProvider.cs

540 lines
21 KiB
C#
Raw Normal View History

2016-03-27 21:11:27 +00:00
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities.TV;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
2017-06-23 16:04:45 +00:00
using MediaBrowser.Common.Progress;
2016-10-25 19:02:04 +00:00
using MediaBrowser.Controller.IO;
using MediaBrowser.Model.IO;
2016-10-24 02:45:23 +00:00
using MediaBrowser.Model.Globalization;
2016-10-27 19:03:23 +00:00
using MediaBrowser.Model.Xml;
2018-09-12 17:26:21 +00:00
using MediaBrowser.Controller.Entities;
namespace MediaBrowser.Providers.TV
{
2016-10-27 07:58:33 +00:00
public class MissingEpisodeProvider
{
private readonly IServerConfigurationManager _config;
private readonly ILogger _logger;
private readonly ILibraryManager _libraryManager;
2014-12-23 03:58:14 +00:00
private readonly ILocalizationManager _localization;
2015-09-13 23:07:54 +00:00
private readonly IFileSystem _fileSystem;
2015-01-14 04:20:30 +00:00
private readonly CultureInfo _usCulture = new CultureInfo("en-US");
2016-10-27 19:03:23 +00:00
private readonly IXmlReaderSettingsFactory _xmlSettings;
2016-10-27 19:03:23 +00:00
public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization, IFileSystem fileSystem, IXmlReaderSettingsFactory xmlSettings)
{
_logger = logger;
_config = config;
_libraryManager = libraryManager;
2014-12-23 03:58:14 +00:00
_localization = localization;
2015-09-13 23:07:54 +00:00
_fileSystem = fileSystem;
2016-10-27 19:03:23 +00:00
_xmlSettings = xmlSettings;
}
2018-09-12 17:26:21 +00:00
public async Task<bool> Run(Series series, bool addNewItems, CancellationToken cancellationToken)
{
2018-09-12 17:26:21 +00:00
var tvdbId = series.GetProviderId(MetadataProviders.Tvdb);
2015-11-24 18:45:52 +00:00
// Todo: Support series by imdb id
var seriesProviderIds = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
seriesProviderIds[MetadataProviders.Tvdb.ToString()] = tvdbId;
var seriesDataPath = TvdbSeriesProvider.GetSeriesDataPath(_config.ApplicationPaths, seriesProviderIds);
2016-12-26 17:38:12 +00:00
// Doesn't have required provider id's
if (string.IsNullOrWhiteSpace(seriesDataPath))
{
2018-09-12 17:26:21 +00:00
return false;
2016-12-26 17:38:12 +00:00
}
2017-01-03 05:15:16 +00:00
// Check this in order to avoid logging an exception due to directory not existing
if (!_fileSystem.DirectoryExists(seriesDataPath))
{
2018-09-12 17:26:21 +00:00
return false;
2017-01-03 05:15:16 +00:00
}
2016-10-27 07:58:33 +00:00
var episodeFiles = _fileSystem.GetFilePaths(seriesDataPath)
.Where(i => string.Equals(Path.GetExtension(i), ".xml", StringComparison.OrdinalIgnoreCase))
.Select(Path.GetFileNameWithoutExtension)
.Where(i => i.StartsWith("episode-", StringComparison.OrdinalIgnoreCase))
.ToList();
var episodeLookup = episodeFiles
.Select(i =>
{
var parts = i.Split('-');
if (parts.Length == 3)
{
int seasonNumber;
2015-01-14 04:20:30 +00:00
if (int.TryParse(parts[1], NumberStyles.Integer, _usCulture, out seasonNumber))
{
int episodeNumber;
2015-01-14 04:20:30 +00:00
if (int.TryParse(parts[2], NumberStyles.Integer, _usCulture, out episodeNumber))
{
2018-09-12 17:26:21 +00:00
return new ValueTuple<int, int>(seasonNumber, episodeNumber);
}
}
}
2018-09-12 17:26:21 +00:00
return new ValueTuple<int, int>(-1, -1);
})
.Where(i => i.Item1 != -1 && i.Item2 != -1)
.ToList();
2018-09-12 17:26:21 +00:00
var allRecursiveChildren = series.GetRecursiveChildren();
var hasBadData = HasInvalidContent(series, allRecursiveChildren);
2014-04-09 02:12:17 +00:00
// Be conservative here to avoid creating missing episodes for ones they already have
2018-09-12 17:26:21 +00:00
var addMissingEpisodes = !hasBadData && _libraryManager.GetLibraryOptions(series).ImportMissingEpisodes;
2018-09-12 17:26:21 +00:00
var anySeasonsRemoved = RemoveObsoleteOrMissingSeasons(series, allRecursiveChildren, episodeLookup);
2018-09-12 17:26:21 +00:00
if (anySeasonsRemoved)
{
// refresh this
allRecursiveChildren = series.GetRecursiveChildren();
}
2018-09-12 17:26:21 +00:00
var anyEpisodesRemoved = RemoveObsoleteOrMissingEpisodes(series, allRecursiveChildren, episodeLookup, addMissingEpisodes);
2018-09-12 17:26:21 +00:00
if (anyEpisodesRemoved)
{
2018-09-12 17:26:21 +00:00
// refresh this
allRecursiveChildren = series.GetRecursiveChildren();
}
2014-03-02 15:42:21 +00:00
2018-09-12 17:26:21 +00:00
var hasNewEpisodes = false;
if (addNewItems && series.IsMetadataFetcherEnabled(_libraryManager.GetLibraryOptions(series) ,TvdbSeriesProvider.Current.Name))
{
hasNewEpisodes = await AddMissingEpisodes(series, allRecursiveChildren, addMissingEpisodes, seriesDataPath, episodeLookup, cancellationToken)
.ConfigureAwait(false);
}
2015-01-14 04:20:30 +00:00
if (hasNewEpisodes || anySeasonsRemoved || anyEpisodesRemoved)
{
2018-09-12 17:26:21 +00:00
return true;
}
2018-09-12 17:26:21 +00:00
return false;
}
2014-04-09 02:12:17 +00:00
/// <summary>
/// Returns true if a series has any seasons or episodes without season or episode numbers
/// If this data is missing no virtual items will be added in order to prevent possible duplicates
/// </summary>
2018-09-12 17:26:21 +00:00
private bool HasInvalidContent(Series series, IList<BaseItem> allItems)
2014-04-09 02:12:17 +00:00
{
return allItems.OfType<Season>().Any(i => !i.IndexNumber.HasValue) ||
allItems.OfType<Episode>().Any(i =>
{
if (!i.ParentIndexNumber.HasValue)
{
return true;
}
// You could have episodes under season 0 with no number
return false;
});
2014-04-09 02:12:17 +00:00
}
private const double UnairedEpisodeThresholdDays = 2;
/// <summary>
/// Adds the missing episodes.
/// </summary>
/// <param name="series">The series.</param>
/// <returns>Task.</returns>
2018-09-12 17:26:21 +00:00
private async Task<bool> AddMissingEpisodes(Series series,
IList<BaseItem> allItems,
bool addMissingEpisodes,
2015-01-25 06:34:50 +00:00
string seriesDataPath,
2018-09-12 17:26:21 +00:00
IEnumerable<ValueTuple<int, int>> episodeLookup,
2014-09-13 14:10:30 +00:00
CancellationToken cancellationToken)
{
2018-09-12 17:26:21 +00:00
var existingEpisodes = allItems.OfType<Episode>()
.ToList();
2018-09-12 17:26:21 +00:00
var lookup = episodeLookup as IList<ValueTuple<int, int>> ?? episodeLookup.ToList();
var seasonCounts = (from e in lookup
2015-01-25 06:34:50 +00:00
group e by e.Item1 into g
select g)
.ToDictionary(g => g.Key, g => g.Count());
var hasChanges = false;
foreach (var tuple in lookup)
{
if (tuple.Item1 <= 0)
{
// Ignore season zeros
continue;
}
if (tuple.Item2 <= 0)
{
// Ignore episode zeros
continue;
}
var existingEpisode = GetExistingEpisode(existingEpisodes, seasonCounts, tuple);
if (existingEpisode != null)
{
continue;
}
var airDate = GetAirDate(seriesDataPath, tuple.Item1, tuple.Item2);
if (!airDate.HasValue)
{
continue;
}
2016-12-08 15:40:20 +00:00
var now = DateTime.UtcNow;
now = now.AddDays(0 - UnairedEpisodeThresholdDays);
2016-12-08 15:40:20 +00:00
if (airDate.Value < now)
{
if (addMissingEpisodes)
2014-09-13 14:10:30 +00:00
{
// tvdb has a lot of nearly blank episodes
2018-09-12 17:26:21 +00:00
_logger.Info("Creating virtual missing episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2);
await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false);
2014-09-13 14:10:30 +00:00
hasChanges = true;
}
}
else if (airDate.Value > now)
{
// tvdb has a lot of nearly blank episodes
2018-09-12 17:26:21 +00:00
_logger.Info("Creating virtual unaired episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2);
await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false);
hasChanges = true;
}
}
return hasChanges;
}
/// <summary>
/// Removes the virtual entry after a corresponding physical version has been added
/// </summary>
2018-09-12 17:26:21 +00:00
private bool RemoveObsoleteOrMissingEpisodes(Series series,
IList<BaseItem> allRecursiveChildren,
IEnumerable<ValueTuple<int, int>> episodeLookup,
bool allowMissingEpisodes)
{
2018-09-12 17:26:21 +00:00
var existingEpisodes = allRecursiveChildren.OfType<Episode>()
.ToList();
var physicalEpisodes = existingEpisodes
2018-09-12 17:26:21 +00:00
.Where(i => i.LocationType != LocationType.Virtual)
.ToList();
var virtualEpisodes = existingEpisodes
2018-09-12 17:26:21 +00:00
.Where(i => i.LocationType == LocationType.Virtual)
.ToList();
var episodesToRemove = virtualEpisodes
.Where(i =>
{
2018-09-12 17:26:21 +00:00
if (i.IndexNumber.HasValue && i.ParentIndexNumber.HasValue)
{
2018-09-12 17:26:21 +00:00
var seasonNumber = i.ParentIndexNumber.Value;
var episodeNumber = i.IndexNumber.Value;
// If there's a physical episode with the same season and episode number, delete it
if (physicalEpisodes.Any(p =>
2018-09-12 17:26:21 +00:00
p.ParentIndexNumber.HasValue && (p.ParentIndexNumber.Value) == seasonNumber &&
p.ContainsEpisodeNumber(episodeNumber)))
{
return true;
}
// If the episode no longer exists in the remote lookup, delete it
if (!episodeLookup.Any(e => e.Item1 == seasonNumber && e.Item2 == episodeNumber))
{
return true;
}
2018-09-12 17:26:21 +00:00
if (!allowMissingEpisodes && i.IsMissingEpisode)
{
// If it's missing, but not unaired, remove it
2018-09-12 17:26:21 +00:00
if (!i.PremiereDate.HasValue || i.PremiereDate.Value.ToLocalTime().Date.AddDays(UnairedEpisodeThresholdDays) < DateTime.Now.Date)
{
return true;
}
}
return false;
}
return true;
})
.ToList();
var hasChanges = false;
2018-09-12 17:26:21 +00:00
foreach (var episodeToRemove in episodesToRemove)
{
2018-09-12 17:26:21 +00:00
_libraryManager.DeleteItem(episodeToRemove, new DeleteOptions
2016-02-12 04:54:00 +00:00
{
DeleteFileLocation = true
2018-09-12 17:26:21 +00:00
}, false);
hasChanges = true;
}
return hasChanges;
}
/// <summary>
/// Removes the obsolete or missing seasons.
/// </summary>
/// <param name="series">The series.</param>
/// <param name="episodeLookup">The episode lookup.</param>
/// <returns>Task{System.Boolean}.</returns>
2018-09-12 17:26:21 +00:00
private bool RemoveObsoleteOrMissingSeasons(Series series,
IList<BaseItem> allRecursiveChildren,
IEnumerable<ValueTuple<int, int>> episodeLookup)
{
2018-09-12 17:26:21 +00:00
var existingSeasons = allRecursiveChildren.OfType<Season>().ToList();
var physicalSeasons = existingSeasons
.Where(i => i.LocationType != LocationType.Virtual)
.ToList();
var virtualSeasons = existingSeasons
.Where(i => i.LocationType == LocationType.Virtual)
.ToList();
2018-09-12 17:26:21 +00:00
var allEpisodes = allRecursiveChildren.OfType<Episode>().ToList();
var seasonsToRemove = virtualSeasons
.Where(i =>
{
if (i.IndexNumber.HasValue)
{
var seasonNumber = i.IndexNumber.Value;
// If there's a physical season with the same number, delete it
if (physicalSeasons.Any(p => p.IndexNumber.HasValue && (p.IndexNumber.Value) == seasonNumber && string.Equals(p.Series.PresentationUniqueKey, i.Series.PresentationUniqueKey, StringComparison.Ordinal)))
{
return true;
}
2018-09-12 17:26:21 +00:00
// If the season no longer exists in the remote lookup, delete it, but only if an existing episode doesn't require it
if (episodeLookup.All(e => e.Item1 != seasonNumber))
{
2018-09-12 17:26:21 +00:00
if (allEpisodes.All(s => s.ParentIndexNumber != seasonNumber || s.IsInSeasonFolder))
{
return true;
}
}
return false;
}
2014-12-23 03:58:14 +00:00
// Season does not have a number
// Remove if there are no episodes directly in series without a season number
2018-09-12 17:26:21 +00:00
return allEpisodes.All(s => s.ParentIndexNumber.HasValue || s.IsInSeasonFolder);
})
.ToList();
var hasChanges = false;
foreach (var seasonToRemove in seasonsToRemove)
{
2018-09-12 17:26:21 +00:00
_libraryManager.DeleteItem(seasonToRemove, new DeleteOptions
2016-02-12 04:54:00 +00:00
{
DeleteFileLocation = true
2018-09-12 17:26:21 +00:00
}, false);
hasChanges = true;
}
return hasChanges;
}
/// <summary>
/// Adds the episode.
/// </summary>
/// <param name="series">The series.</param>
/// <param name="seasonNumber">The season number.</param>
/// <param name="episodeNumber">The episode number.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private async Task AddEpisode(Series series, int seasonNumber, int episodeNumber, CancellationToken cancellationToken)
{
var season = series.Children.OfType<Season>()
.FirstOrDefault(i => i.IndexNumber.HasValue && i.IndexNumber.Value == seasonNumber);
if (season == null)
{
2015-09-13 23:07:54 +00:00
var provider = new DummySeasonProvider(_config, _logger, _localization, _libraryManager, _fileSystem);
2016-05-20 19:45:04 +00:00
season = await provider.AddSeason(series, seasonNumber, true, cancellationToken).ConfigureAwait(false);
}
2015-01-14 04:20:30 +00:00
var name = string.Format("Episode {0}", episodeNumber.ToString(_usCulture));
var episode = new Episode
{
Name = name,
IndexNumber = episodeNumber,
ParentIndexNumber = seasonNumber,
2016-06-12 05:03:52 +00:00
Id = _libraryManager.GetNewItemId((series.Id + seasonNumber.ToString(_usCulture) + name), typeof(Episode)),
2016-07-05 05:40:18 +00:00
IsVirtualItem = true,
2018-09-12 17:26:21 +00:00
SeasonId = season == null ? Guid.Empty : season.Id,
2016-07-05 06:01:31 +00:00
SeriesId = series.Id
};
2015-07-09 05:52:25 +00:00
episode.SetParent(season);
season.AddChild(episode, cancellationToken);
await episode.RefreshMetadata(new MetadataRefreshOptions(_fileSystem), cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Gets the existing episode.
/// </summary>
/// <param name="existingEpisodes">The existing episodes.</param>
/// <param name="seasonCounts"></param>
/// <param name="tuple">The tuple.</param>
/// <returns>Episode.</returns>
2018-09-12 17:26:21 +00:00
private Episode GetExistingEpisode(IList<Episode> existingEpisodes, Dictionary<int, int> seasonCounts, ValueTuple<int, int> tuple)
{
var s = tuple.Item1;
var e = tuple.Item2;
while (true)
{
var episode = GetExistingEpisode(existingEpisodes, s, e);
if (episode != null)
return episode;
s--;
if (seasonCounts.ContainsKey(s))
e += seasonCounts[s];
else
break;
}
return null;
}
2018-09-12 17:26:21 +00:00
private Episode GetExistingEpisode(IEnumerable<Episode> existingEpisodes, int season, int episode)
{
return existingEpisodes
2018-09-12 17:26:21 +00:00
.FirstOrDefault(i => i.ParentIndexNumber == season && i.ContainsEpisodeNumber(episode));
}
/// <summary>
/// Gets the air date.
/// </summary>
/// <param name="seriesDataPath">The series data path.</param>
/// <param name="seasonNumber">The season number.</param>
/// <param name="episodeNumber">The episode number.</param>
/// <returns>System.Nullable{DateTime}.</returns>
private DateTime? GetAirDate(string seriesDataPath, int seasonNumber, int episodeNumber)
{
// First open up the tvdb xml file and make sure it has valid data
2015-01-14 04:20:30 +00:00
var filename = string.Format("episode-{0}-{1}.xml", seasonNumber.ToString(_usCulture), episodeNumber.ToString(_usCulture));
var xmlPath = Path.Combine(seriesDataPath, filename);
DateTime? airDate = null;
2016-10-27 19:03:23 +00:00
using (var fileStream = _fileSystem.GetFileStream(xmlPath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read))
{
2016-10-27 19:03:23 +00:00
// It appears the best way to filter out invalid entries is to only include those with valid air dates
using (var streamReader = new StreamReader(fileStream, Encoding.UTF8))
{
2016-10-27 19:03:23 +00:00
var settings = _xmlSettings.Create(false);
settings.CheckCharacters = false;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreComments = true;
2016-10-27 19:03:23 +00:00
// Use XmlReader for best performance
using (var reader = XmlReader.Create(streamReader, settings))
{
2016-10-27 19:03:23 +00:00
reader.MoveToContent();
2016-11-02 17:29:06 +00:00
reader.Read();
2016-10-27 19:03:23 +00:00
// Loop through each element
2016-12-03 21:46:06 +00:00
while (!reader.EOF && reader.ReadState == ReadState.Interactive)
{
2016-10-27 19:03:23 +00:00
if (reader.NodeType == XmlNodeType.Element)
{
2016-10-27 19:03:23 +00:00
switch (reader.Name)
{
case "EpisodeName":
{
2016-10-27 19:03:23 +00:00
var val = reader.ReadElementContentAsString();
if (string.IsNullOrWhiteSpace(val))
{
// Not valid, ignore these
return null;
}
break;
}
2016-10-27 19:03:23 +00:00
case "FirstAired":
{
2016-10-27 19:03:23 +00:00
var val = reader.ReadElementContentAsString();
if (!string.IsNullOrWhiteSpace(val))
{
2016-10-27 19:03:23 +00:00
DateTime date;
if (DateTime.TryParse(val, out date))
{
airDate = date.ToUniversalTime();
}
}
2016-10-27 19:03:23 +00:00
break;
}
2016-10-27 19:03:23 +00:00
default:
2016-11-02 17:29:06 +00:00
{
reader.Skip();
break;
}
2016-10-27 19:03:23 +00:00
}
}
2016-11-02 17:29:06 +00:00
else
{
reader.Read();
}
}
}
}
}
return airDate;
}
}
}