ease up on person info downloading
This commit is contained in:
parent
c0bbf8945f
commit
9938b39ac2
|
@ -86,7 +86,6 @@
|
||||||
<Compile Include="Movies\MovieProviderFromXml.cs" />
|
<Compile Include="Movies\MovieProviderFromXml.cs" />
|
||||||
<Compile Include="Movies\OpenMovieDatabaseProvider.cs" />
|
<Compile Include="Movies\OpenMovieDatabaseProvider.cs" />
|
||||||
<Compile Include="Movies\PersonProviderFromXml.cs" />
|
<Compile Include="Movies\PersonProviderFromXml.cs" />
|
||||||
<Compile Include="Movies\PersonUpdatesPreScanTask.cs" />
|
|
||||||
<Compile Include="Movies\MovieDbPersonProvider.cs" />
|
<Compile Include="Movies\MovieDbPersonProvider.cs" />
|
||||||
<Compile Include="Music\AlbumInfoFromSongProvider.cs" />
|
<Compile Include="Music\AlbumInfoFromSongProvider.cs" />
|
||||||
<Compile Include="Music\AlbumProviderFromXml.cs" />
|
<Compile Include="Music\AlbumProviderFromXml.cs" />
|
||||||
|
|
|
@ -46,43 +46,26 @@ namespace MediaBrowser.Providers.Movies
|
||||||
return images.Where(i => i.Type == imageType);
|
return images.Where(i => i.Type == imageType);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<IEnumerable<RemoteImageInfo>> GetAllImages(IHasImages item, CancellationToken cancellationToken)
|
public async Task<IEnumerable<RemoteImageInfo>> GetAllImages(IHasImages item, CancellationToken cancellationToken)
|
||||||
{
|
|
||||||
return GetAllImagesInternal(item, true, cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<IEnumerable<RemoteImageInfo>> GetAllImagesInternal(IHasImages item, bool retryOnMissingData, CancellationToken cancellationToken)
|
|
||||||
{
|
{
|
||||||
var person = (Person)item;
|
var person = (Person)item;
|
||||||
var id = person.GetProviderId(MetadataProviders.Tmdb);
|
var id = person.GetProviderId(MetadataProviders.Tmdb);
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(id))
|
if (!string.IsNullOrEmpty(id))
|
||||||
{
|
{
|
||||||
|
await MovieDbPersonProvider.Current.DownloadPersonInfoIfNeeded(id, cancellationToken).ConfigureAwait(false);
|
||||||
|
|
||||||
var dataFilePath = MovieDbPersonProvider.GetPersonDataFilePath(_config.ApplicationPaths, id);
|
var dataFilePath = MovieDbPersonProvider.GetPersonDataFilePath(_config.ApplicationPaths, id);
|
||||||
|
|
||||||
try
|
var result = _jsonSerializer.DeserializeFromFile<MovieDbPersonProvider.PersonResult>(dataFilePath);
|
||||||
{
|
|
||||||
var result = _jsonSerializer.DeserializeFromFile<MovieDbPersonProvider.PersonResult>(dataFilePath);
|
|
||||||
|
|
||||||
var images = result.images ?? new MovieDbPersonProvider.Images();
|
var images = result.images ?? new MovieDbPersonProvider.Images();
|
||||||
|
|
||||||
var tmdbSettings = await MovieDbProvider.Current.GetTmdbSettings(cancellationToken).ConfigureAwait(false);
|
var tmdbSettings = await MovieDbProvider.Current.GetTmdbSettings(cancellationToken).ConfigureAwait(false);
|
||||||
|
|
||||||
var tmdbImageUrl = tmdbSettings.images.base_url + "original";
|
var tmdbImageUrl = tmdbSettings.images.base_url + "original";
|
||||||
|
|
||||||
return GetImages(images, tmdbImageUrl);
|
return GetImages(images, tmdbImageUrl);
|
||||||
}
|
|
||||||
catch (FileNotFoundException)
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retryOnMissingData)
|
|
||||||
{
|
|
||||||
await MovieDbPersonProvider.Current.DownloadPersonInfo(id, cancellationToken).ConfigureAwait(false);
|
|
||||||
|
|
||||||
return await GetAllImagesInternal(item, false, cancellationToken).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return new List<RemoteImageInfo>();
|
return new List<RemoteImageInfo>();
|
||||||
|
|
|
@ -86,7 +86,7 @@ namespace MediaBrowser.Providers.Movies
|
||||||
|
|
||||||
protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo)
|
protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo)
|
||||||
{
|
{
|
||||||
if (HasAltMeta(item) && !ConfigurationManager.Configuration.EnableTmdbUpdates)
|
if (HasAltMeta(item))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return base.NeedsRefreshInternal(item, providerInfo);
|
return base.NeedsRefreshInternal(item, providerInfo);
|
||||||
|
@ -235,17 +235,12 @@ namespace MediaBrowser.Providers.Movies
|
||||||
/// <returns>Task.</returns>
|
/// <returns>Task.</returns>
|
||||||
private async Task FetchInfo(Person person, string id, bool isForcedRefresh, CancellationToken cancellationToken)
|
private async Task FetchInfo(Person person, string id, bool isForcedRefresh, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var dataFilePath = GetPersonDataFilePath(ConfigurationManager.ApplicationPaths, id);
|
await DownloadPersonInfoIfNeeded(id, cancellationToken).ConfigureAwait(false);
|
||||||
|
|
||||||
// Only download if not already there
|
if (isForcedRefresh || !HasAltMeta(person))
|
||||||
// The prescan task will take care of updates so we don't need to re-download here
|
|
||||||
if (!File.Exists(dataFilePath))
|
|
||||||
{
|
{
|
||||||
await DownloadPersonInfo(id, cancellationToken).ConfigureAwait(false);
|
var dataFilePath = GetPersonDataFilePath(ConfigurationManager.ApplicationPaths, id);
|
||||||
}
|
|
||||||
|
|
||||||
if (isForcedRefresh || ConfigurationManager.Configuration.EnableTmdbUpdates || !HasAltMeta(person))
|
|
||||||
{
|
|
||||||
var info = JsonSerializer.DeserializeFromFile<PersonResult>(dataFilePath);
|
var info = JsonSerializer.DeserializeFromFile<PersonResult>(dataFilePath);
|
||||||
|
|
||||||
cancellationToken.ThrowIfCancellationRequested();
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
@ -254,10 +249,17 @@ namespace MediaBrowser.Providers.Movies
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal async Task DownloadPersonInfo(string id, CancellationToken cancellationToken)
|
internal async Task DownloadPersonInfoIfNeeded(string id, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var personDataPath = GetPersonDataPath(ConfigurationManager.ApplicationPaths, id);
|
var personDataPath = GetPersonDataPath(ConfigurationManager.ApplicationPaths, id);
|
||||||
|
|
||||||
|
var fileInfo = _fileSystem.GetFileSystemInfo(personDataPath);
|
||||||
|
|
||||||
|
if (fileInfo.Exists && (DateTime.UtcNow - _fileSystem.GetLastWriteTimeUtc(fileInfo)).TotalDays <= 7)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
var url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
|
var url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
|
||||||
|
|
||||||
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||||
|
|
|
@ -1,236 +0,0 @@
|
||||||
using MediaBrowser.Common.IO;
|
|
||||||
using MediaBrowser.Common.Net;
|
|
||||||
using MediaBrowser.Controller.Configuration;
|
|
||||||
using MediaBrowser.Controller.Library;
|
|
||||||
using MediaBrowser.Model.Logging;
|
|
||||||
using MediaBrowser.Model.Serialization;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Globalization;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace MediaBrowser.Providers.Movies
|
|
||||||
{
|
|
||||||
public class PersonUpdatesPreScanTask : IPeoplePrescanTask
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The updates URL
|
|
||||||
/// </summary>
|
|
||||||
private const string UpdatesUrl = "http://api.themoviedb.org/3/person/changes?start_date={0}&api_key={1}&page={2}";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The _HTTP client
|
|
||||||
/// </summary>
|
|
||||||
private readonly IHttpClient _httpClient;
|
|
||||||
/// <summary>
|
|
||||||
/// The _logger
|
|
||||||
/// </summary>
|
|
||||||
private readonly ILogger _logger;
|
|
||||||
/// <summary>
|
|
||||||
/// The _config
|
|
||||||
/// </summary>
|
|
||||||
private readonly IServerConfigurationManager _config;
|
|
||||||
private readonly IJsonSerializer _json;
|
|
||||||
private readonly IFileSystem _fileSystem;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new instance of the <see cref="PersonUpdatesPreScanTask"/> class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="logger">The logger.</param>
|
|
||||||
/// <param name="httpClient">The HTTP client.</param>
|
|
||||||
/// <param name="config">The config.</param>
|
|
||||||
public PersonUpdatesPreScanTask(ILogger logger, IHttpClient httpClient, IServerConfigurationManager config, IJsonSerializer json, IFileSystem fileSystem)
|
|
||||||
{
|
|
||||||
_logger = logger;
|
|
||||||
_httpClient = httpClient;
|
|
||||||
_config = config;
|
|
||||||
_json = json;
|
|
||||||
_fileSystem = fileSystem;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected readonly CultureInfo UsCulture = new CultureInfo("en-US");
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Runs the specified progress.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="progress">The progress.</param>
|
|
||||||
/// <param name="cancellationToken">The cancellation token.</param>
|
|
||||||
/// <returns>Task.</returns>
|
|
||||||
public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
if (!_config.Configuration.EnableInternetProviders || !_config.Configuration.EnableTmdbUpdates)
|
|
||||||
{
|
|
||||||
progress.Report(100);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var path = MovieDbPersonProvider.GetPersonsDataPath(_config.CommonApplicationPaths);
|
|
||||||
|
|
||||||
Directory.CreateDirectory(path);
|
|
||||||
|
|
||||||
var timestampFile = Path.Combine(path, "time.txt");
|
|
||||||
|
|
||||||
var timestampFileInfo = new FileInfo(timestampFile);
|
|
||||||
|
|
||||||
// Don't check for updates every single time
|
|
||||||
if (timestampFileInfo.Exists && (DateTime.UtcNow - _fileSystem.GetLastWriteTimeUtc(timestampFileInfo)).TotalDays < 10)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find out the last time we queried tvdb for updates
|
|
||||||
var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
|
|
||||||
|
|
||||||
var existingDirectories = GetExistingIds(path).ToList();
|
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(lastUpdateTime))
|
|
||||||
{
|
|
||||||
long lastUpdateTicks;
|
|
||||||
|
|
||||||
if (long.TryParse(lastUpdateTime, NumberStyles.Any, UsCulture, out lastUpdateTicks))
|
|
||||||
{
|
|
||||||
var lastUpdateDate = new DateTime(lastUpdateTicks, DateTimeKind.Utc);
|
|
||||||
|
|
||||||
// They only allow up to 14 days of updates
|
|
||||||
if ((DateTime.UtcNow - lastUpdateDate).TotalDays > 13)
|
|
||||||
{
|
|
||||||
lastUpdateDate = DateTime.UtcNow.AddDays(-13);
|
|
||||||
}
|
|
||||||
|
|
||||||
var updatedIds = await GetIdsToUpdate(lastUpdateDate, 1, cancellationToken).ConfigureAwait(false);
|
|
||||||
|
|
||||||
var existingDictionary = existingDirectories.ToDictionary(i => i, StringComparer.OrdinalIgnoreCase);
|
|
||||||
|
|
||||||
var idsToUpdate = updatedIds.Where(i => !string.IsNullOrWhiteSpace(i) && existingDictionary.ContainsKey(i));
|
|
||||||
|
|
||||||
await UpdatePeople(idsToUpdate, progress, cancellationToken).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
File.WriteAllText(timestampFile, DateTime.UtcNow.Ticks.ToString(UsCulture), Encoding.UTF8);
|
|
||||||
progress.Report(100);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the existing ids.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="path">The path.</param>
|
|
||||||
/// <returns>IEnumerable{System.String}.</returns>
|
|
||||||
private IEnumerable<string> GetExistingIds(string path)
|
|
||||||
{
|
|
||||||
return Directory.EnumerateDirectories(path)
|
|
||||||
.SelectMany(Directory.EnumerateDirectories)
|
|
||||||
.Select(Path.GetFileNameWithoutExtension);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the ids to update.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="lastUpdateTime">The last update time.</param>
|
|
||||||
/// <param name="page">The page.</param>
|
|
||||||
/// <param name="cancellationToken">The cancellation token.</param>
|
|
||||||
/// <returns>Task{IEnumerable{System.String}}.</returns>
|
|
||||||
private async Task<IEnumerable<string>> GetIdsToUpdate(DateTime lastUpdateTime, int page, CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
var hasMorePages = false;
|
|
||||||
var list = new List<string>();
|
|
||||||
|
|
||||||
// First get last time
|
|
||||||
using (var stream = await _httpClient.Get(new HttpRequestOptions
|
|
||||||
{
|
|
||||||
Url = string.Format(UpdatesUrl, lastUpdateTime.ToString("yyyy-MM-dd"), MovieDbProvider.ApiKey, page),
|
|
||||||
CancellationToken = cancellationToken,
|
|
||||||
EnableHttpCompression = true,
|
|
||||||
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
|
|
||||||
AcceptHeader = MovieDbProvider.AcceptHeader
|
|
||||||
|
|
||||||
}).ConfigureAwait(false))
|
|
||||||
{
|
|
||||||
var obj = _json.DeserializeFromStream<RootObject>(stream);
|
|
||||||
|
|
||||||
var data = obj.results.Select(i => i.id.ToString(UsCulture));
|
|
||||||
|
|
||||||
list.AddRange(data);
|
|
||||||
|
|
||||||
hasMorePages = page < obj.total_pages;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (hasMorePages)
|
|
||||||
{
|
|
||||||
var more = await GetIdsToUpdate(lastUpdateTime, page + 1, cancellationToken).ConfigureAwait(false);
|
|
||||||
|
|
||||||
list.AddRange(more);
|
|
||||||
}
|
|
||||||
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Updates the people.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="ids">The ids.</param>
|
|
||||||
/// <param name="progress">The progress.</param>
|
|
||||||
/// <param name="cancellationToken">The cancellation token.</param>
|
|
||||||
/// <returns>Task.</returns>
|
|
||||||
private async Task UpdatePeople(IEnumerable<string> ids, IProgress<double> progress, CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
var list = ids.ToList();
|
|
||||||
var numComplete = 0;
|
|
||||||
|
|
||||||
foreach (var id in list)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await UpdatePerson(id, cancellationToken).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.ErrorException("Error updating tmdb person id {0}", ex, id);
|
|
||||||
}
|
|
||||||
|
|
||||||
numComplete++;
|
|
||||||
double percent = numComplete;
|
|
||||||
percent /= list.Count;
|
|
||||||
percent *= 100;
|
|
||||||
|
|
||||||
progress.Report(percent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Updates the person.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="id">The id.</param>
|
|
||||||
/// <param name="cancellationToken">The cancellation token.</param>
|
|
||||||
/// <returns>Task.</returns>
|
|
||||||
private Task UpdatePerson(string id, CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
_logger.Info("Updating person from tmdb " + id);
|
|
||||||
|
|
||||||
return MovieDbPersonProvider.Current.DownloadPersonInfo(id, cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
class Result
|
|
||||||
{
|
|
||||||
public int id { get; set; }
|
|
||||||
public bool? adult { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
class RootObject
|
|
||||||
{
|
|
||||||
public List<Result> results { get; set; }
|
|
||||||
public int page { get; set; }
|
|
||||||
public int total_pages { get; set; }
|
|
||||||
public int total_results { get; set; }
|
|
||||||
|
|
||||||
public RootObject()
|
|
||||||
{
|
|
||||||
results = new List<Result>();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user