lgladsden/jellyfin-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

limit number of tmdb requests per second

Browse Source
This commit is contained in:
Luke Pulverenti 2013-05-19 14:42:58 -04:00
parent 13d997a97a
commit 25314e1fc9
7 changed files with 87 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MediaBrowser.Controller/Providers/IProviderManager.cs
View File

@ -21,6 +21,8 @@ namespace MediaBrowser.Controller.Providers
/// <exception cref="System.ArgumentNullException">item</exception>
Task<string> DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken);
Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken);
/// <summary>
/// Saves to library filesystem.
/// </summary>

31
MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs
View File

@ -207,11 +207,10 @@ namespace MediaBrowser.Controller.Providers.Movies
/// <returns>Task{MovieImages}.</returns>
private async Task<MovieImages> FetchImages(BaseItem item, string id, CancellationToken cancellationToken)
{
using (var json = await _httpClient.Get(new HttpRequestOptions
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
Url = string.Format(GetImages, id, MovieDbProvider.ApiKey, item is BoxSet ? "collection" : "movie"),
CancellationToken = cancellationToken,
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
AcceptHeader = MovieDbProvider.AcceptHeader,
EnableResponseCache = true
@ -264,14 +263,14 @@ namespace MediaBrowser.Controller.Providers.Movies
}
if (poster != null)
{
try
var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
item.PrimaryImagePath = await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + poster.file_path, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false);
}
catch (HttpException)
{
status = ProviderRefreshStatus.CompletedWithErrors;
}
Url = tmdbImageUrl + poster.file_path,
CancellationToken = cancellationToken
}).ConfigureAwait(false);
item.PrimaryImagePath = await _providerManager.SaveImage(item, img, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false);
}
}
@ -295,14 +294,14 @@ namespace MediaBrowser.Controller.Providers.Movies
if (ConfigurationManager.Configuration.RefreshItemImages || !hasLocalBackdrop)
{
try
var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + images.backdrops[i].file_path, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false));
}
catch (HttpException)
{
status = ProviderRefreshStatus.CompletedWithErrors;
}
Url = tmdbImageUrl + images.backdrops[i].file_path,
CancellationToken = cancellationToken
}).ConfigureAwait(false);
item.BackdropImagePaths.Add(await _providerManager.SaveImage(item, img, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false));
}
}
}

61
MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs
View File

@ -40,7 +40,7 @@ namespace MediaBrowser.Controller.Providers.Movies
/// <summary>
/// The movie db
/// </summary>
internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1,1);
private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1,1);
internal static MovieDbProvider Current { get; private set; }
@ -81,7 +81,7 @@ namespace MediaBrowser.Controller.Providers.Movies
{
if (dispose)
{
MovieDbResourcePool.Dispose();
_movieDbResourcePool.Dispose();
}
}
@ -172,7 +172,7 @@ namespace MediaBrowser.Controller.Providers.Movies
{
get
{
LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer, HttpClient));
LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer));
return _tmdbSettingsTask;
}
}
@ -181,15 +181,14 @@ namespace MediaBrowser.Controller.Providers.Movies
/// Gets the TMDB settings.
/// </summary>
/// <returns>Task{TmdbSettingsResult}.</returns>
private static async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer, IHttpClient httpClient)
private async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer)
{
try
{
using (var json = await httpClient.Get(new HttpRequestOptions
using (var json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = string.Format(TmdbConfigUrl, ApiKey),
CancellationToken = CancellationToken.None,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -560,11 +559,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (Stream json = await HttpClient.Get(new HttpRequestOptions
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url3,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -600,11 +598,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (var json = await HttpClient.Get(new HttpRequestOptions
using (var json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url3,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -647,11 +644,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (var json = await HttpClient.Get(new HttpRequestOptions
using (var json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url3,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -737,11 +733,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (Stream json = await HttpClient.Get(new HttpRequestOptions
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -822,11 +817,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (var json = await HttpClient.Get(new HttpRequestOptions
using (var json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -863,11 +857,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (Stream json = await HttpClient.Get(new HttpRequestOptions
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = Current.MovieDbResourcePool,
AcceptHeader = AcceptHeader,
EnableResponseCache = true
@ -1022,6 +1015,38 @@ namespace MediaBrowser.Controller.Providers.Movies
}
private DateTime _lastRequestDate = DateTime.MinValue;
/// <summary>
/// Gets the movie db response.
/// </summary>
internal async Task<Stream> GetMovieDbResponse(HttpRequestOptions options)
{
var cancellationToken = options.CancellationToken;
await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
// Limit to three requests per second
var diff = 330 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
if (diff > 0)
{
await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
}
_lastRequestDate = DateTime.Now;
return await HttpClient.Get(options).ConfigureAwait(false);
}
finally
{
_lastRequestDate = DateTime.Now;
_movieDbResourcePool.Release();
}
}
/// <summary>
/// The remove

7
MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs
View File

@ -1,5 +1,4 @@
using MediaBrowser.Common.Net;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Serialization;
@ -15,8 +14,8 @@ namespace MediaBrowser.Controller.Providers.Movies
/// </summary>
class PersonProviderFromJson : TmdbPersonProvider
{
public PersonProviderFromJson(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
: base(httpClient, jsonSerializer, logManager, configurationManager, providerManager)
public PersonProviderFromJson(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
: base(jsonSerializer, logManager, configurationManager, providerManager)
{
}

26
MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs
View File

@ -28,18 +28,13 @@ namespace MediaBrowser.Controller.Providers.Movies
protected readonly IProviderManager ProviderManager;
public TmdbPersonProvider(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
: base(logManager, configurationManager)
{
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
if (httpClient == null)
{
throw new ArgumentNullException("httpClient");
}
HttpClient = httpClient;
JsonSerializer = jsonSerializer;
ProviderManager = providerManager;
}
@ -50,12 +45,6 @@ namespace MediaBrowser.Controller.Providers.Movies
/// <value>The json serializer.</value>
protected IJsonSerializer JsonSerializer { get; private set; }
/// <summary>
/// Gets the HTTP client.
/// </summary>
/// <value>The HTTP client.</value>
protected IHttpClient HttpClient { get; private set; }
/// <summary>
/// Supportses the specified item.
/// </summary>
@ -172,11 +161,10 @@ namespace MediaBrowser.Controller.Providers.Movies
try
{
using (Stream json = await HttpClient.Get(new HttpRequestOptions
using (Stream json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
AcceptHeader = MovieDbProvider.AcceptHeader,
EnableResponseCache = true
@ -204,11 +192,10 @@ namespace MediaBrowser.Controller.Providers.Movies
string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
PersonResult searchResult = null;
using (var json = await HttpClient.Get(new HttpRequestOptions
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
AcceptHeader = MovieDbProvider.AcceptHeader,
EnableResponseCache = true
@ -340,7 +327,12 @@ namespace MediaBrowser.Controller.Providers.Movies
var localPath = Path.Combine(item.MetaLocation, targetName);
if (!item.ResolveArgs.ContainsMetaFileByName(targetName))
{
using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false))
using (var sourceStream = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
{
Url = source,
CancellationToken = cancellationToken
}).ConfigureAwait(false))
{
await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false);

11
MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs
View File

@ -206,9 +206,7 @@ namespace MediaBrowser.Controller.Providers.Music
/// <summary>
/// The _last music brainz request
/// </summary>
private DateTime _lastMusicBrainzRequest = DateTime.MinValue;
private readonly SemaphoreSlim _musicBrainzSemaphore = new SemaphoreSlim(1, 1);
private DateTime _lastRequestDate = DateTime.MinValue;
/// <summary>
/// Gets the music brainz response.
@ -222,7 +220,7 @@ namespace MediaBrowser.Controller.Providers.Music
try
{
var diff = 1500 - (DateTime.Now - _lastMusicBrainzRequest).TotalMilliseconds;
var diff = 1500 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
// MusicBrainz is extremely adamant about limiting to one request per second
@ -231,7 +229,7 @@ namespace MediaBrowser.Controller.Providers.Music
await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
}
_lastMusicBrainzRequest = DateTime.Now;
_lastRequestDate = DateTime.Now;
var doc = new XmlDocument();
@ -239,7 +237,6 @@ namespace MediaBrowser.Controller.Providers.Music
{
Url = url,
CancellationToken = cancellationToken,
ResourcePool = _musicBrainzSemaphore,
UserAgent = "MediaBrowserServer/www.mediabrowser3.com",
EnableResponseCache = true
@ -255,7 +252,7 @@ namespace MediaBrowser.Controller.Providers.Music
}
finally
{
_lastMusicBrainzRequest = DateTime.Now;
_lastRequestDate = DateTime.Now;
_musicBrainzResourcePool.Release();
}

16
MediaBrowser.Server.Implementations/Providers/ProviderManager.cs
View File

@ -357,16 +357,22 @@ namespace MediaBrowser.Server.Implementations.Providers
throw new ArgumentNullException("resourcePool");
}
var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
//download and save locally
return await SaveImage(item, img, targetName, saveLocally, cancellationToken).ConfigureAwait(false);
}
public async Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken)
{
//download and save locally
var localPath = (saveLocally && item.MetaLocation != null) ?
Path.Combine(item.MetaLocation, targetName) :
_remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName);
var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
if (saveLocally) // queue to media directories
{
await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false);
await SaveToLibraryFilesystem(item, localPath, source, cancellationToken).ConfigureAwait(false);
}
else
{
@ -376,7 +382,7 @@ namespace MediaBrowser.Server.Implementations.Providers
{
using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
{
await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
await source.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
@ -390,7 +396,7 @@ namespace MediaBrowser.Server.Implementations.Providers
}
finally
{
img.Dispose();
source.Dispose();
}
}