jellyfin-server/MediaBrowser.Providers/TV/TheTVDB/TvdbPrescanTask.cs

401 lines
16 KiB
C#
Raw Normal View History

2016-03-27 21:11:27 +00:00
using MediaBrowser.Common.Net;
2013-05-25 17:55:32 +00:00
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities.TV;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Net;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
2017-05-21 07:25:49 +00:00
using MediaBrowser.Controller.Dto;
2016-10-25 19:02:04 +00:00
using MediaBrowser.Model.IO;
2016-05-06 06:02:10 +00:00
using MediaBrowser.Controller.Entities;
2016-10-25 19:02:04 +00:00
using MediaBrowser.Controller.IO;
2016-10-27 19:03:23 +00:00
using MediaBrowser.Model.Xml;
2013-06-09 16:47:28 +00:00
namespace MediaBrowser.Providers.TV
{
/// <summary>
/// Class TvdbPrescanTask
/// </summary>
public class TvdbPrescanTask : ILibraryPostScanTask
{
/// <summary>
/// The server time URL
/// </summary>
2016-04-21 05:13:33 +00:00
private const string ServerTimeUrl = "https://thetvdb.com/api/Updates.php?type=none";
/// <summary>
/// The updates URL
/// </summary>
2016-04-21 05:13:33 +00:00
private const string UpdatesUrl = "https://thetvdb.com/api/Updates.php?type=all&time={0}";
/// <summary>
/// The _HTTP client
/// </summary>
private readonly IHttpClient _httpClient;
/// <summary>
/// The _logger
/// </summary>
private readonly ILogger _logger;
/// <summary>
/// The _config
/// </summary>
2013-05-25 17:55:32 +00:00
private readonly IServerConfigurationManager _config;
private readonly IFileSystem _fileSystem;
private readonly ILibraryManager _libraryManager;
2016-10-27 19:03:23 +00:00
private readonly IXmlReaderSettingsFactory _xmlSettings;
/// <summary>
/// Initializes a new instance of the <see cref="TvdbPrescanTask"/> class.
/// </summary>
/// <param name="logger">The logger.</param>
/// <param name="httpClient">The HTTP client.</param>
/// <param name="config">The config.</param>
2016-10-27 19:03:23 +00:00
public TvdbPrescanTask(ILogger logger, IHttpClient httpClient, IServerConfigurationManager config, IFileSystem fileSystem, ILibraryManager libraryManager, IXmlReaderSettingsFactory xmlSettings)
{
_logger = logger;
_httpClient = httpClient;
_config = config;
_fileSystem = fileSystem;
_libraryManager = libraryManager;
2016-10-27 19:03:23 +00:00
_xmlSettings = xmlSettings;
}
protected readonly CultureInfo UsCulture = new CultureInfo("en-US");
/// <summary>
/// Runs the specified progress.
/// </summary>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
{
2014-02-12 04:29:13 +00:00
var seriesConfig = _config.Configuration.MetadataOptions.FirstOrDefault(i => string.Equals(i.ItemType, typeof(Series).Name, StringComparison.OrdinalIgnoreCase));
if (seriesConfig != null && seriesConfig.DisabledMetadataFetchers.Contains(TvdbSeriesProvider.Current.Name, StringComparer.OrdinalIgnoreCase))
2013-05-25 17:55:32 +00:00
{
progress.Report(100);
return;
}
var path = TvdbSeriesProvider.GetSeriesDataPath(_config.CommonApplicationPaths);
2016-05-06 06:02:10 +00:00
_fileSystem.CreateDirectory(path);
var timestampFile = Path.Combine(path, "time.txt");
2015-10-04 03:38:46 +00:00
var timestampFileInfo = _fileSystem.GetFileInfo(timestampFile);
// Don't check for tvdb updates anymore frequently than 24 hours
if (timestampFileInfo.Exists && (DateTime.UtcNow - _fileSystem.GetLastWriteTimeUtc(timestampFileInfo)).TotalDays < 1)
{
return;
}
// Find out the last time we queried tvdb for updates
2016-05-06 06:02:10 +00:00
var lastUpdateTime = timestampFileInfo.Exists ? _fileSystem.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
string newUpdateTime;
2016-10-27 19:03:23 +00:00
var existingDirectories = _fileSystem.GetDirectoryPaths(path)
2014-02-10 18:39:41 +00:00
.Select(Path.GetFileName)
.ToList();
2016-05-06 06:02:10 +00:00
var seriesList = _libraryManager.GetItemList(new InternalItemsQuery()
{
IncludeItemTypes = new[] { typeof(Series).Name },
Recursive = true,
2017-05-21 07:25:49 +00:00
GroupByPresentationUniqueKey = false,
DtoOptions = new DtoOptions(false)
{
EnableImages = false
}
}).Cast<Series>()
.ToList();
2016-05-06 06:02:10 +00:00
var seriesIdsInLibrary = seriesList
.Where(i => !string.IsNullOrEmpty(i.GetProviderId(MetadataProviders.Tvdb)))
2014-02-10 18:39:41 +00:00
.Select(i => i.GetProviderId(MetadataProviders.Tvdb))
.ToList();
var missingSeries = seriesIdsInLibrary.Except(existingDirectories, StringComparer.OrdinalIgnoreCase)
.ToList();
2016-05-06 06:02:10 +00:00
var enableInternetProviders = seriesList.Count == 0 ? false : seriesList[0].IsInternetMetadataEnabled();
if (!enableInternetProviders)
{
progress.Report(100);
return;
}
// If this is our first time, update all series
if (string.IsNullOrEmpty(lastUpdateTime))
{
// First get tvdb server time
using (var stream = await _httpClient.Get(new HttpRequestOptions
{
Url = ServerTimeUrl,
CancellationToken = cancellationToken,
EnableHttpCompression = true,
2017-01-05 18:09:12 +00:00
BufferContent = false
}).ConfigureAwait(false))
{
2013-09-15 17:34:48 +00:00
newUpdateTime = GetUpdateTime(stream);
}
2014-02-10 18:39:41 +00:00
existingDirectories.AddRange(missingSeries);
await UpdateSeries(existingDirectories, path, null, progress, cancellationToken).ConfigureAwait(false);
}
else
{
var seriesToUpdate = await GetSeriesIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false);
newUpdateTime = seriesToUpdate.Item2;
long lastUpdateValue;
long.TryParse(lastUpdateTime, NumberStyles.Any, UsCulture, out lastUpdateValue);
var nullableUpdateValue = lastUpdateValue == 0 ? (long?)null : lastUpdateValue;
2014-02-10 18:39:41 +00:00
var listToUpdate = seriesToUpdate.Item1.ToList();
listToUpdate.AddRange(missingSeries);
await UpdateSeries(listToUpdate, path, nullableUpdateValue, progress, cancellationToken).ConfigureAwait(false);
}
2016-05-06 06:02:10 +00:00
_fileSystem.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8);
progress.Report(100);
}
2013-09-15 17:34:48 +00:00
/// <summary>
/// Gets the update time.
/// </summary>
/// <param name="response">The response.</param>
/// <returns>System.String.</returns>
private string GetUpdateTime(Stream response)
{
2016-10-27 19:03:23 +00:00
var settings = _xmlSettings.Create(false);
settings.CheckCharacters = false;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreComments = true;
2013-09-15 17:34:48 +00:00
using (var streamReader = new StreamReader(response, Encoding.UTF8))
{
// Use XmlReader for best performance
using (var reader = XmlReader.Create(streamReader, settings))
{
reader.MoveToContent();
2016-11-02 17:08:20 +00:00
reader.Read();
2013-09-15 17:34:48 +00:00
// Loop through each element
2016-12-03 21:46:06 +00:00
while (!reader.EOF && reader.ReadState == ReadState.Interactive)
2013-09-15 17:34:48 +00:00
{
if (reader.NodeType == XmlNodeType.Element)
{
switch (reader.Name)
{
case "Time":
{
return (reader.ReadElementContentAsString() ?? string.Empty).Trim();
}
default:
reader.Skip();
break;
}
}
2016-11-02 17:08:20 +00:00
else
{
reader.Read();
}
2013-09-15 17:34:48 +00:00
}
}
}
return null;
}
/// <summary>
/// Gets the series ids to update.
/// </summary>
/// <param name="existingSeriesIds">The existing series ids.</param>
/// <param name="lastUpdateTime">The last update time.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task{IEnumerable{System.String}}.</returns>
private async Task<Tuple<IEnumerable<string>, string>> GetSeriesIdsToUpdate(IEnumerable<string> existingSeriesIds, string lastUpdateTime, CancellationToken cancellationToken)
{
// First get last time
using (var stream = await _httpClient.Get(new HttpRequestOptions
{
Url = string.Format(UpdatesUrl, lastUpdateTime),
CancellationToken = cancellationToken,
EnableHttpCompression = true,
2017-01-05 18:09:12 +00:00
BufferContent = false
}).ConfigureAwait(false))
{
2013-09-15 17:34:48 +00:00
var data = GetUpdatedSeriesIdList(stream);
2013-09-15 17:34:48 +00:00
var existingDictionary = existingSeriesIds.ToDictionary(i => i, StringComparer.OrdinalIgnoreCase);
2013-09-15 17:34:48 +00:00
var seriesList = data.Item1
.Where(i => !string.IsNullOrWhiteSpace(i) && existingDictionary.ContainsKey(i));
2013-09-15 17:34:48 +00:00
return new Tuple<IEnumerable<string>, string>(seriesList, data.Item2);
}
}
2013-09-15 17:34:48 +00:00
private Tuple<List<string>, string> GetUpdatedSeriesIdList(Stream stream)
{
string updateTime = null;
var idList = new List<string>();
2013-08-07 16:00:20 +00:00
2016-10-27 19:03:23 +00:00
var settings = _xmlSettings.Create(false);
settings.CheckCharacters = false;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreComments = true;
2013-09-15 17:34:48 +00:00
using (var streamReader = new StreamReader(stream, Encoding.UTF8))
{
// Use XmlReader for best performance
using (var reader = XmlReader.Create(streamReader, settings))
{
reader.MoveToContent();
2016-11-02 17:08:20 +00:00
reader.Read();
2013-09-15 17:34:48 +00:00
// Loop through each element
2016-12-03 21:46:06 +00:00
while (!reader.EOF && reader.ReadState == ReadState.Interactive)
2013-09-15 17:34:48 +00:00
{
if (reader.NodeType == XmlNodeType.Element)
{
switch (reader.Name)
{
case "Time":
{
updateTime = (reader.ReadElementContentAsString() ?? string.Empty).Trim();
break;
}
case "Series":
{
var id = (reader.ReadElementContentAsString() ?? string.Empty).Trim();
idList.Add(id);
break;
}
default:
reader.Skip();
break;
}
}
2016-11-02 17:08:20 +00:00
else
{
reader.Read();
}
2013-09-15 17:34:48 +00:00
}
}
}
2013-09-15 17:34:48 +00:00
return new Tuple<List<string>, string>(idList, updateTime);
}
/// <summary>
/// Updates the series.
/// </summary>
/// <param name="seriesIds">The series ids.</param>
/// <param name="seriesDataPath">The series data path.</param>
/// <param name="lastTvDbUpdateTime">The last tv db update time.</param>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private async Task UpdateSeries(List<string> seriesIds, string seriesDataPath, long? lastTvDbUpdateTime, IProgress<double> progress, CancellationToken cancellationToken)
{
var numComplete = 0;
2016-05-06 06:02:10 +00:00
var seriesList = _libraryManager.GetItemList(new InternalItemsQuery()
{
IncludeItemTypes = new[] { typeof(Series).Name },
Recursive = true,
2017-05-21 07:25:49 +00:00
GroupByPresentationUniqueKey = false,
DtoOptions = new DtoOptions(false)
{
EnableImages = false
}
2016-05-06 06:02:10 +00:00
}).Cast<Series>();
// Gather all series into a lookup by tvdb id
2016-05-06 06:02:10 +00:00
var allSeries = seriesList
.Where(i => !string.IsNullOrEmpty(i.GetProviderId(MetadataProviders.Tvdb)))
.ToLookup(i => i.GetProviderId(MetadataProviders.Tvdb));
foreach (var seriesId in seriesIds)
{
// Find the preferred language(s) for the movie in the library
var languages = allSeries[seriesId]
.Select(i => i.GetPreferredMetadataLanguage())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
foreach (var language in languages)
{
try
{
await UpdateSeries(seriesId, seriesDataPath, lastTvDbUpdateTime, language, cancellationToken).ConfigureAwait(false);
}
catch (HttpException ex)
{
_logger.ErrorException("Error updating tvdb series id {0}, language {1}", ex, seriesId, language);
2016-05-06 06:02:10 +00:00
// Already logged at lower levels, but don't fail the whole operation, unless timed out
// We have to fail this to make it run again otherwise new episode data could potentially be missing
if (ex.IsTimedOut)
{
throw;
}
}
}
numComplete++;
double percent = numComplete;
percent /= seriesIds.Count;
percent *= 100;
progress.Report(percent);
}
}
/// <summary>
/// Updates the series.
/// </summary>
/// <param name="id">The id.</param>
/// <param name="seriesDataPath">The series data path.</param>
/// <param name="lastTvDbUpdateTime">The last tv db update time.</param>
/// <param name="preferredMetadataLanguage">The preferred metadata language.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private Task UpdateSeries(string id, string seriesDataPath, long? lastTvDbUpdateTime, string preferredMetadataLanguage, CancellationToken cancellationToken)
{
2014-02-03 17:44:13 +00:00
_logger.Info("Updating series from tvdb " + id + ", language " + preferredMetadataLanguage);
seriesDataPath = Path.Combine(seriesDataPath, id);
2016-05-06 06:02:10 +00:00
_fileSystem.CreateDirectory(seriesDataPath);
2013-09-15 17:34:48 +00:00
2017-03-12 19:27:45 +00:00
return TvdbSeriesProvider.Current.DownloadSeriesZip(id, MetadataProviders.Tvdb.ToString(), null, null, seriesDataPath, lastTvDbUpdateTime, preferredMetadataLanguage, cancellationToken);
}
}
}