Merge pull request #2723 from MediaBrowser/dev

Dev

Emby.Common.Implementations/ScheduledTasks/ScheduledTaskWorker.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Events;
 using MediaBrowser.Common.Extensions;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.Events;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Logging;
@@ -402,7 +403,7 @@ namespace Emby.Common.Implementations.ScheduledTasks
                 throw new InvalidOperationException("Cannot execute a Task that is already running");
             }
 
-            var progress = new Progress<double>();
+            var progress = new SimpleProgress<double>();
 
             CurrentCancellationTokenSource = new CancellationTokenSource();
 

Emby.Drawing.ImageMagick/PlayedIndicatorDrawer.cs

@@ -5,7 +5,7 @@ using MediaBrowser.Model.Drawing;
 using System;
 using System.IO;
 using System.Threading.Tasks;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.IO;
 
@@ -104,7 +104,7 @@ namespace Emby.Drawing.ImageMagick
             var tempPath = await httpClient.GetTempFile(new HttpRequestOptions
             {
                 Url = url,
-                Progress = new Progress<double>()
+                Progress = new SimpleProgress<double>()
 
             }).ConfigureAwait(false);
 

Emby.Drawing.Skia/PlayedIndicatorDrawer.cs

@@ -9,6 +9,7 @@ using System.Threading.Tasks;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.IO;
 using System.Reflection;
+using MediaBrowser.Common.Progress;
 
 namespace Emby.Drawing.Skia
 {
@@ -99,7 +100,7 @@ namespace Emby.Drawing.Skia
             var tempPath = await httpClient.GetTempFile(new HttpRequestOptions
             {
                 Url = url,
-                Progress = new Progress<double>()
+                Progress = new SimpleProgress<double>()
 
             }).ConfigureAwait(false);
 

Emby.Server.Implementations/Channels/ChannelManager.cs

@@ -23,7 +23,7 @@ using System.Linq;
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Controller.Entities.Audio;
 using MediaBrowser.Controller.Entities.Movies;
@@ -980,7 +980,7 @@ namespace Emby.Server.Implementations.Channels
                 ? null
                 : _userManager.GetUserById(query.UserId);
 
-            var internalResult = await GetChannelItemsInternal(query, new Progress<double>(), cancellationToken).ConfigureAwait(false);
+            var internalResult = await GetChannelItemsInternal(query, new SimpleProgress<double>(), cancellationToken).ConfigureAwait(false);
 
             var dtoOptions = new DtoOptions()
             {

Emby.Server.Implementations/Channels/ChannelPostScanTask.cs

@@ -207,7 +207,7 @@ namespace Emby.Server.Implementations.Channels
                     StartIndex = totalRetrieved,
                     FolderId = folderId
 
-                }, new Progress<double>(), cancellationToken);
+                }, new SimpleProgress<double>(), cancellationToken);
 
                 folderItems.AddRange(result.Items.Where(i => i.IsFolder).Select(i => i.Id.ToString("N")));
 

Emby.Server.Implementations/Channels/RefreshChannelsScheduledTask.cs

@@ -4,6 +4,7 @@ using MediaBrowser.Model.Logging;
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.Tasks;
 
 namespace Emby.Server.Implementations.Channels
@@ -42,7 +43,7 @@ namespace Emby.Server.Implementations.Channels
         {
             var manager = (ChannelManager)_channelManager;
 
-            await manager.RefreshChannels(new Progress<double>(), cancellationToken).ConfigureAwait(false);
+            await manager.RefreshChannels(new SimpleProgress<double>(), cancellationToken).ConfigureAwait(false);
 
             await new ChannelPostScanTask(_channelManager, _userManager, _logger, _libraryManager).Run(progress, cancellationToken)
                     .ConfigureAwait(false);

Emby.Server.Implementations/Dto/DtoService.cs

@@ -105,9 +105,13 @@ namespace Emby.Server.Implementations.Dto
             var programTuples = new List<Tuple<BaseItem, BaseItemDto>>();
             var channelTuples = new List<Tuple<BaseItemDto, LiveTvChannel>>();
 
+            var refreshQueue = options.Fields.Contains(ItemFields.RefreshState)
+                ? _providerManager.GetRefreshQueue()
+                : null;
+
             foreach (var item in items)
             {
-                var dto = GetBaseItemDtoInternal(item, options, user, owner);
+                var dto = GetBaseItemDtoInternal(item, options, refreshQueue, user, owner);
 
                 var tvChannel = item as LiveTvChannel;
                 if (tvChannel != null)
@@ -160,7 +164,11 @@ namespace Emby.Server.Implementations.Dto
         {
             var syncDictionary = GetSyncedItemProgress(options);
 
-            var dto = GetBaseItemDtoInternal(item, options, user, owner);
+            var refreshQueue = options.Fields.Contains(ItemFields.RefreshState)
+                ? _providerManager.GetRefreshQueue()
+                : null;
+
+            var dto = GetBaseItemDtoInternal(item, options, refreshQueue, user, owner);
             var tvChannel = item as LiveTvChannel;
             if (tvChannel != null)
             {
@@ -292,7 +300,7 @@ namespace Emby.Server.Implementations.Dto
             }
         }
 
-        private BaseItemDto GetBaseItemDtoInternal(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null)
+        private BaseItemDto GetBaseItemDtoInternal(BaseItem item, DtoOptions options, Dictionary<Guid, Guid> currentRefreshQueue, User user = null, BaseItem owner = null)
         {
             var fields = options.Fields;
 
@@ -392,6 +400,11 @@ namespace Emby.Server.Implementations.Dto
                 dto.Etag = item.GetEtag(user);
             }
 
+            if (currentRefreshQueue != null)
+            {
+                //dto.RefreshState = item.GetRefreshState(currentRefreshQueue);
+            }
+
             if (item is ILiveTvRecording)
             {
                 _livetvManager().AddInfoToRecordingDto(item, dto, user);
@@ -402,7 +415,10 @@ namespace Emby.Server.Implementations.Dto
 
         public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List<BaseItem> taggedItems, Dictionary<string, SyncedItemProgress> syncProgress, User user = null)
         {
-            var dto = GetBaseItemDtoInternal(item, options, user);
+            var refreshQueue = options.Fields.Contains(ItemFields.RefreshState)
+                ? _providerManager.GetRefreshQueue()
+                : null;
+            var dto = GetBaseItemDtoInternal(item, options, refreshQueue, user);
 
             if (taggedItems != null && options.Fields.Contains(ItemFields.ItemCounts))
             {

Emby.Server.Implementations/Emby.Server.Implementations.csproj

@@ -148,7 +148,6 @@
     <Compile Include="Library\Validators\PeopleValidator.cs" />
     <Compile Include="Library\Validators\StudiosPostScanTask.cs" />
     <Compile Include="Library\Validators\StudiosValidator.cs" />
-    <Compile Include="Library\Validators\YearsPostScanTask.cs" />
     <Compile Include="LiveTv\ChannelImageProvider.cs" />
     <Compile Include="LiveTv\EmbyTV\DirectRecorder.cs" />
     <Compile Include="LiveTv\EmbyTV\EmbyTV.cs" />

Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs

@@ -6,9 +6,14 @@ using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Logging;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Threading;
+using MediaBrowser.Controller.Channels;
 using MediaBrowser.Controller.Entities.Audio;
+using MediaBrowser.Controller.Entities.TV;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model.Events;
 using MediaBrowser.Model.Extensions;
 using MediaBrowser.Model.Threading;
 
@@ -49,13 +54,16 @@ namespace Emby.Server.Implementations.EntryPoints
         /// </summary>
         private const int LibraryUpdateDuration = 5000;
 
-        public LibraryChangedNotifier(ILibraryManager libraryManager, ISessionManager sessionManager, IUserManager userManager, ILogger logger, ITimerFactory timerFactory)
+        private readonly IProviderManager _providerManager;
+
+        public LibraryChangedNotifier(ILibraryManager libraryManager, ISessionManager sessionManager, IUserManager userManager, ILogger logger, ITimerFactory timerFactory, IProviderManager providerManager)
         {
             _libraryManager = libraryManager;
             _sessionManager = sessionManager;
             _userManager = userManager;
             _logger = logger;
             _timerFactory = timerFactory;
+            _providerManager = providerManager;
         }
 
         public void Run()
@@ -64,6 +72,108 @@ namespace Emby.Server.Implementations.EntryPoints
             _libraryManager.ItemUpdated += libraryManager_ItemUpdated;
             _libraryManager.ItemRemoved += libraryManager_ItemRemoved;
 
+            _providerManager.RefreshCompleted += _providerManager_RefreshCompleted;
+            _providerManager.RefreshStarted += _providerManager_RefreshStarted;
+            _providerManager.RefreshProgress += _providerManager_RefreshProgress;
+        }
+
+        private Dictionary<Guid, DateTime> _lastProgressMessageTimes = new Dictionary<Guid, DateTime>();
+
+        private void _providerManager_RefreshProgress(object sender, GenericEventArgs<Tuple<BaseItem, double>> e)
+        {
+            var item = e.Argument.Item1;
+
+            if (!EnableRefreshMessage(item))
+            {
+                return;
+            }
+
+            var progress = e.Argument.Item2;
+
+            DateTime lastMessageSendTime;
+            if (_lastProgressMessageTimes.TryGetValue(item.Id, out lastMessageSendTime))
+            {
+                if (progress > 0 && progress < 100 && (DateTime.UtcNow - lastMessageSendTime).TotalMilliseconds < 1000)
+                {
+                    return;
+                }
+            }
+
+            _lastProgressMessageTimes[item.Id] = DateTime.UtcNow;
+
+            var dict = new Dictionary<string, string>();
+            dict["ItemId"] = item.Id.ToString("N");
+            dict["Progress"] = progress.ToString(CultureInfo.InvariantCulture);
+
+            try
+            {
+                _sessionManager.SendMessageToAdminSessions("RefreshProgress", dict, CancellationToken.None);
+
+                _logger.Info("Sending refresh progress {0} {1}", item.Id.ToString("N"), progress);
+            }
+            catch
+            {
+            }
+
+            var collectionFolders = _libraryManager.GetCollectionFolders(item).ToList();
+
+            foreach (var collectionFolder in collectionFolders)
+            {
+                var collectionFolderDict = new Dictionary<string, string>();
+                collectionFolderDict["ItemId"] = collectionFolder.Id.ToString("N");
+                collectionFolderDict["Progress"] = (collectionFolder.GetRefreshProgress() ?? 0).ToString(CultureInfo.InvariantCulture);
+
+                try
+                {
+                    _sessionManager.SendMessageToAdminSessions("RefreshProgress", collectionFolderDict, CancellationToken.None);
+                }
+                catch
+                {
+
+                }
+            }
+        }
+
+        private void _providerManager_RefreshStarted(object sender, GenericEventArgs<BaseItem> e)
+        {
+            _providerManager_RefreshProgress(sender, new GenericEventArgs<Tuple<BaseItem, double>>(new Tuple<BaseItem, double>(e.Argument, 0)));
+        }
+
+        private void _providerManager_RefreshCompleted(object sender, GenericEventArgs<BaseItem> e)
+        {
+            _providerManager_RefreshProgress(sender, new GenericEventArgs<Tuple<BaseItem, double>>(new Tuple<BaseItem, double>(e.Argument, 100)));
+        }
+
+        private bool EnableRefreshMessage(BaseItem item)
+        {
+            var folder = item as Folder;
+
+            if (folder == null)
+            {
+                return false;
+            }
+
+            if (folder.IsRoot)
+            {
+                return false;
+            }
+
+            if (folder is AggregateFolder || folder is UserRootFolder)
+            {
+                return false;
+            }
+
+            if (folder is UserView || folder is Channel)
+            {
+                return false;
+            }
+
+            if (!folder.IsTopParent)
+            {
+                return false;
+            }
+
+            return true;
         }
 
         /// <summary>

Emby.Server.Implementations/IO/FileRefresher.cs

@@ -6,7 +6,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Common.Events;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.IO;
@@ -170,7 +170,7 @@ namespace Emby.Server.Implementations.IO
             // If the root folder changed, run the library task so the user can see it
             if (itemsToRefresh.Any(i => i is AggregateFolder))
             {
-                LibraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                LibraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                 return;
             }
 

Emby.Server.Implementations/Library/LibraryManager.cs

@@ -462,7 +462,7 @@ namespace Emby.Server.Implementations.Library
 
                 if (parent != null)
                 {
-                    await parent.ValidateChildren(new Progress<double>(), CancellationToken.None, new MetadataRefreshOptions(_fileSystem), false).ConfigureAwait(false);
+                    await parent.ValidateChildren(new SimpleProgress<double>(), CancellationToken.None, new MetadataRefreshOptions(_fileSystem), false).ConfigureAwait(false);
                 }
             }
             else if (parent != null)
@@ -1113,13 +1113,13 @@ namespace Emby.Server.Implementations.Library
             progress.Report(.5);
 
             // Start by just validating the children of the root, but go no further
-            await RootFolder.ValidateChildren(new Progress<double>(), cancellationToken, new MetadataRefreshOptions(_fileSystem), recursive: false);
+            await RootFolder.ValidateChildren(new SimpleProgress<double>(), cancellationToken, new MetadataRefreshOptions(_fileSystem), recursive: false);
 
             progress.Report(1);
 
             await GetUserRootFolder().RefreshMetadata(cancellationToken).ConfigureAwait(false);
 
-            await GetUserRootFolder().ValidateChildren(new Progress<double>(), cancellationToken, new MetadataRefreshOptions(_fileSystem), recursive: false).ConfigureAwait(false);
+            await GetUserRootFolder().ValidateChildren(new SimpleProgress<double>(), cancellationToken, new MetadataRefreshOptions(_fileSystem), recursive: false).ConfigureAwait(false);
             progress.Report(2);
 
             // Quickly scan CollectionFolders for changes
@@ -1204,25 +1204,24 @@ namespace Emby.Server.Implementations.Library
         /// Gets the default view.
         /// </summary>
         /// <returns>IEnumerable{VirtualFolderInfo}.</returns>
-        public IEnumerable<VirtualFolderInfo> GetVirtualFolders()
+        public List<VirtualFolderInfo> GetVirtualFolders()
         {
-            return GetView(ConfigurationManager.ApplicationPaths.DefaultUserViewsPath);
+            return GetVirtualFolders(false);
         }
 
-        /// <summary>
+        public List<VirtualFolderInfo> GetVirtualFolders(bool includeRefreshState)
-        /// Gets the view.
-        /// </summary>
-        /// <param name="path">The path.</param>
-        /// <returns>IEnumerable{VirtualFolderInfo}.</returns>
-        private IEnumerable<VirtualFolderInfo> GetView(string path)
         {
             var topLibraryFolders = GetUserRootFolder().Children.ToList();
 
-            return _fileSystem.GetDirectoryPaths(path)
+            var refreshQueue = includeRefreshState ? _providerManagerFactory().GetRefreshQueue() : null;
-                .Select(dir => GetVirtualFolderInfo(dir, topLibraryFolders));
+
+            return _fileSystem.GetDirectoryPaths(ConfigurationManager.ApplicationPaths.DefaultUserViewsPath)
+                .Select(dir => GetVirtualFolderInfo(dir, topLibraryFolders, refreshQueue))
+                .OrderBy(i => i.Name)
+                .ToList();
         }
 
-        private VirtualFolderInfo GetVirtualFolderInfo(string dir, List<BaseItem> allCollectionFolders)
+        private VirtualFolderInfo GetVirtualFolderInfo(string dir, List<BaseItem> allCollectionFolders, Dictionary<Guid, Guid> refreshQueue)
         {
             var info = new VirtualFolderInfo
             {
@@ -1248,6 +1247,13 @@ namespace Emby.Server.Implementations.Library
             {
                 info.ItemId = libraryFolder.Id.ToString("N");
                 info.LibraryOptions = GetLibraryOptions(libraryFolder);
+
+                if (refreshQueue != null)
+                {
+                    info.RefreshProgress = libraryFolder.GetRefreshProgress();
+
+                    info.RefreshStatus = info.RefreshProgress.HasValue ? "Active" : refreshQueue.ContainsKey(libraryFolder.Id) ? "Queued" : "Idle";
+                }
             }
 
             return info;
@@ -2947,7 +2953,7 @@ namespace Emby.Server.Implementations.Library
                     // No need to start if scanning the library because it will handle it
                     if (refreshLibrary)
                     {
-                        ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                        ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                     }
                     else
                     {
@@ -3075,7 +3081,7 @@ namespace Emby.Server.Implementations.Library
         private void SyncLibraryOptionsToLocations(string virtualFolderPath, LibraryOptions options)
         {
             var topLibraryFolders = GetUserRootFolder().Children.ToList();
-            var info = GetVirtualFolderInfo(virtualFolderPath, topLibraryFolders);
+            var info = GetVirtualFolderInfo(virtualFolderPath, topLibraryFolders, null);
 
             if (info.Locations.Count > 0 && info.Locations.Count != options.PathInfos.Length)
             {
@@ -3125,7 +3131,7 @@ namespace Emby.Server.Implementations.Library
                     // No need to start if scanning the library because it will handle it
                     if (refreshLibrary)
                     {
-                        ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                        ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                     }
                     else
                     {

Emby.Server.Implementations/Library/Validators/PeopleValidator.cs

@@ -55,10 +55,6 @@ namespace Emby.Server.Implementations.Library.Validators
         /// <returns>Task.</returns>
         public async Task ValidatePeople(CancellationToken cancellationToken, IProgress<double> progress)
         {
-            var innerProgress = new ActionableProgress<double>();
-
-            innerProgress.RegisterAction(pct => progress.Report(pct * .15));
-
             var people = _libraryManager.GetPeople(new InternalPeopleQuery());
 
             var dict = new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);

Emby.Server.Implementations/Library/Validators/YearsPostScanTask.cs

@@ -1,57 +0,0 @@
-using MediaBrowser.Controller.Library;
-using MediaBrowser.Model.Logging;
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Emby.Server.Implementations.Library.Validators
-{
-    public class YearsPostScanTask : ILibraryPostScanTask
-    {
-        private readonly ILibraryManager _libraryManager;
-        private readonly ILogger _logger;
-
-        public YearsPostScanTask(ILibraryManager libraryManager, ILogger logger)
-        {
-            _libraryManager = libraryManager;
-            _logger = logger;
-        }
-
-        public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
-        {
-            var yearNumber = 1900;
-            var maxYear = DateTime.UtcNow.Year + 3;
-            var count = maxYear - yearNumber + 1;
-            var numComplete = 0;
-
-            while (yearNumber < maxYear)
-            {
-                cancellationToken.ThrowIfCancellationRequested();
-
-                try
-                {
-                    var year = _libraryManager.GetYear(yearNumber);
-
-                    await year.RefreshMetadata(cancellationToken).ConfigureAwait(false);
-                }
-                catch (OperationCanceledException)
-                {
-                    // Don't clutter the log
-                    throw;
-                }
-                catch (Exception ex)
-                {
-                    _logger.ErrorException("Error refreshing year {0}", ex, yearNumber);
-                }
-
-                numComplete++;
-                double percent = numComplete;
-                percent /= count;
-                percent *= 100;
-
-                progress.Report(percent);
-                yearNumber++;
-            }
-        }
-    }
-}

Emby.Server.Implementations/LiveTv/EmbyTV/EmbyTV.cs

@@ -28,7 +28,7 @@ using System.Xml;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Common.Events;
 using MediaBrowser.Common.Extensions;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller;
 using MediaBrowser.Controller.Dto;
 using MediaBrowser.Controller.Entities;
@@ -240,7 +240,7 @@ namespace Emby.Server.Implementations.LiveTv.EmbyTV
 
             if (requiresRefresh)
             {
-                _libraryManager.ValidateMediaLibrary(new Progress<Double>(), CancellationToken.None);
+                _libraryManager.ValidateMediaLibrary(new SimpleProgress<Double>(), CancellationToken.None);
             }
         }
 

Emby.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs

@@ -15,6 +15,7 @@ using Emby.XmlTv.Classes;
 using Emby.XmlTv.Entities;
 using MediaBrowser.Common.Extensions;
 using MediaBrowser.Common.Net;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Logging;
@@ -75,7 +76,7 @@ namespace Emby.Server.Implementations.LiveTv.Listings
             {
                 CancellationToken = cancellationToken,
                 Url = path,
-                Progress = new Progress<Double>(),
+                Progress = new SimpleProgress<Double>(),
                 DecompressionMethod = CompressionMethod.Gzip,
 
                 // It's going to come back gzipped regardless of this value

Emby.Server.Implementations/LiveTv/LiveTvManager.cs

@@ -1273,8 +1273,8 @@ namespace Emby.Server.Implementations.LiveTv
 
             if (coreService != null)
             {
-                await coreService.RefreshSeriesTimers(cancellationToken, new Progress<double>()).ConfigureAwait(false);
+                await coreService.RefreshSeriesTimers(cancellationToken, new SimpleProgress<double>()).ConfigureAwait(false);
-                await coreService.RefreshTimers(cancellationToken, new Progress<double>()).ConfigureAwait(false);
+                await coreService.RefreshTimers(cancellationToken, new SimpleProgress<double>()).ConfigureAwait(false);
             }
 
             // Load these now which will prefetch metadata
@@ -1549,7 +1549,7 @@ namespace Emby.Server.Implementations.LiveTv
 
                 var idList = await Task.WhenAll(recordingTasks).ConfigureAwait(false);
 
-                await CleanDatabaseInternal(idList.ToList(), new[] { typeof(LiveTvVideoRecording).Name, typeof(LiveTvAudioRecording).Name }, new Progress<double>(), cancellationToken).ConfigureAwait(false);
+                await CleanDatabaseInternal(idList.ToList(), new[] { typeof(LiveTvVideoRecording).Name, typeof(LiveTvAudioRecording).Name }, new SimpleProgress<double>(), cancellationToken).ConfigureAwait(false);
 
                 _lastRecordingRefreshTime = DateTime.UtcNow;
             }

Emby.Server.Implementations/News/NewsEntryPoint.cs

@@ -15,6 +15,7 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Threading;
 
@@ -82,7 +83,7 @@ namespace Emby.Server.Implementations.News
             var requestOptions = new HttpRequestOptions
             {
                 Url = "http://emby.media/community/index.php?/blog/rss/1-media-browser-developers-blog",
-                Progress = new Progress<double>(),
+                Progress = new SimpleProgress<double>(),
                 UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.42 Safari/537.36",
                 BufferContent = false
             };

Emby.Server.Implementations/Notifications/WebSocketNotifier.cs

@@ -23,7 +23,6 @@ namespace Emby.Server.Implementations.Notifications
         public void Run()
         {
             _notificationsRepo.NotificationAdded += _notificationsRepo_NotificationAdded;
-
             _notificationsRepo.NotificationsMarkedRead += _notificationsRepo_NotificationsMarkedRead;
         }
 

Emby.Server.Implementations/ScheduledTasks/PluginUpdateTask.cs

@@ -8,6 +8,7 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.Tasks;
 
 namespace Emby.Server.Implementations.ScheduledTasks
@@ -77,7 +78,7 @@ namespace Emby.Server.Implementations.ScheduledTasks
 
                 try
                 {
-                    await _installationManager.InstallPackage(i, true, new Progress<double>(), cancellationToken).ConfigureAwait(false);
+                    await _installationManager.InstallPackage(i, true, new SimpleProgress<double>(), cancellationToken).ConfigureAwait(false);
                 }
                 catch (OperationCanceledException)
                 {

Emby.Server.Implementations/ScheduledTasks/SystemUpdateTask.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.Tasks;
 
 namespace Emby.Server.Implementations.ScheduledTasks
@@ -70,7 +71,7 @@ namespace Emby.Server.Implementations.ScheduledTasks
             EventHandler<double> innerProgressHandler = (sender, e) => progress.Report(e * .1);
 
             // Create a progress object for the update check
-            var innerProgress = new Progress<double>();
+            var innerProgress = new SimpleProgress<double>();
             innerProgress.ProgressChanged += innerProgressHandler;
 
             var updateInfo = await _appHost.CheckForApplicationUpdate(cancellationToken, innerProgress).ConfigureAwait(false);
@@ -97,7 +98,7 @@ namespace Emby.Server.Implementations.ScheduledTasks
 
                 innerProgressHandler = (sender, e) => progress.Report(e * .9 + .1);
 
-                innerProgress = new Progress<double>();
+                innerProgress = new SimpleProgress<double>();
                 innerProgress.ProgressChanged += innerProgressHandler;
 
                 await _appHost.UpdateApplication(updateInfo.Package, cancellationToken, innerProgress).ConfigureAwait(false);

Emby.Server.Implementations/Updates/InstallationManager.cs

@@ -246,7 +246,7 @@ namespace Emby.Server.Implementations.Updates
                 {
                     Url = "https://www.mb3admin.com/admin/service/MB3Packages.json",
                     CancellationToken = cancellationToken,
-                    Progress = new Progress<Double>()
+                    Progress = new SimpleProgress<Double>()
 
                 }).ConfigureAwait(false);
 

MediaBrowser.Api/ItemRefreshService.cs

@@ -65,7 +65,7 @@ namespace MediaBrowser.Api
             _providerManager.QueueRefresh(item.Id, options, RefreshPriority.High);
         }
 
-        private MetadataRefreshOptions GetRefreshOptions(BaseRefreshRequest request)
+        private MetadataRefreshOptions GetRefreshOptions(RefreshItem request)
         {
             return new MetadataRefreshOptions(new DirectoryService(_logger, _fileSystem))
             {
@@ -73,8 +73,9 @@ namespace MediaBrowser.Api
                 ImageRefreshMode = request.ImageRefreshMode,
                 ReplaceAllImages = request.ReplaceAllImages,
                 ReplaceAllMetadata = request.ReplaceAllMetadata,
-                ForceSave = true,
+                ForceSave = request.MetadataRefreshMode == MetadataRefreshMode.FullRefresh || request.ImageRefreshMode == ImageRefreshMode.FullRefresh || request.ReplaceAllImages || request.ReplaceAllMetadata,
-                IsAutomated = false
+                IsAutomated = false,
+                ValidateChildren = request.Recursive
             };
         }
     }

MediaBrowser.Api/Library/LibraryService.cs

@@ -28,6 +28,7 @@ using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.Globalization;
 using MediaBrowser.Model.Services;
 using MediaBrowser.Common.Extensions;
+using MediaBrowser.Common.Progress;
 
 namespace MediaBrowser.Api.Library
 {
@@ -445,7 +446,7 @@ namespace MediaBrowser.Api.Library
             }
             else
             {
-                Task.Run(() => _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None));
+                Task.Run(() => _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None));
             }
         }
 
@@ -483,7 +484,7 @@ namespace MediaBrowser.Api.Library
             }
             else
             {
-                Task.Run(() => _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None));
+                Task.Run(() => _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None));
             }
         }
 
@@ -696,7 +697,7 @@ namespace MediaBrowser.Api.Library
             {
                 try
                 {
-                    _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                    _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                 }
                 catch (Exception ex)
                 {

MediaBrowser.Api/Library/LibraryStructureService.cs

@@ -8,7 +8,7 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
@@ -208,7 +208,7 @@ namespace MediaBrowser.Api.Library
         /// <returns>System.Object.</returns>
         public object Get(GetVirtualFolders request)
         {
-            var result = _libraryManager.GetVirtualFolders().OrderBy(i => i.Name).ToList();
+            var result = _libraryManager.GetVirtualFolders(true);
 
             return ToOptimizedSerializedResultUsingCache(result);
         }
@@ -290,7 +290,7 @@ namespace MediaBrowser.Api.Library
                     // No need to start if scanning the library because it will handle it
                     if (request.RefreshLibrary)
                     {
-                        _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                        _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                     }
                     else
                     {
@@ -347,7 +347,7 @@ namespace MediaBrowser.Api.Library
                     // No need to start if scanning the library because it will handle it
                     if (request.RefreshLibrary)
                     {
-                        _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                        _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                     }
                     else
                     {
@@ -400,7 +400,7 @@ namespace MediaBrowser.Api.Library
                     // No need to start if scanning the library because it will handle it
                     if (request.RefreshLibrary)
                     {
-                        _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None);
+                        _libraryManager.ValidateMediaLibrary(new SimpleProgress<double>(), CancellationToken.None);
                     }
                     else
                     {

MediaBrowser.Api/PackageService.cs

@@ -8,6 +8,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.Services;
 
 namespace MediaBrowser.Api
@@ -156,7 +157,7 @@ namespace MediaBrowser.Api
 
             else if (string.Equals(request.PackageType, "System", StringComparison.OrdinalIgnoreCase) || string.Equals(request.PackageType, "All", StringComparison.OrdinalIgnoreCase))
             {
-                var updateCheckResult = _appHost.CheckForApplicationUpdate(CancellationToken.None, new Progress<double>()).Result;
+                var updateCheckResult = _appHost.CheckForApplicationUpdate(CancellationToken.None, new SimpleProgress<double>()).Result;
 
                 if (updateCheckResult.IsUpdateAvailable)
                 {
@@ -233,7 +234,7 @@ namespace MediaBrowser.Api
                 throw new ResourceNotFoundException(string.Format("Package not found: {0}", request.Name));
             }
 
-            Task.Run(() => _installationManager.InstallPackage(package, true, new Progress<double>(), CancellationToken.None));
+            Task.Run(() => _installationManager.InstallPackage(package, true, new SimpleProgress<double>(), CancellationToken.None));
         }
 
         /// <summary>

MediaBrowser.Common/Extensions/ResourceNotFoundException.cs

@@ -25,4 +25,25 @@ namespace MediaBrowser.Common.Extensions
 
         }
     }
+
+    public class RateLimitExceededException : Exception
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RateLimitExceededException" /> class.
+        /// </summary>
+        public RateLimitExceededException()
+        {
+
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RateLimitExceededException" /> class.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        public RateLimitExceededException(string message)
+            : base(message)
+        {
+
+        }
+    }
 }

MediaBrowser.Common/Progress/ActionableProgress.cs

@@ -7,12 +7,13 @@ namespace MediaBrowser.Common.Progress
     /// Class ActionableProgress
     /// </summary>
     /// <typeparam name="T"></typeparam>
-    public class ActionableProgress<T> : Progress<T>, IDisposable
+    public class ActionableProgress<T> : IProgress<T>, IDisposable
     {
         /// <summary>
         /// The _actions
         /// </summary>
         private readonly List<Action<T>> _actions = new List<Action<T>>();
+        public event EventHandler<T> ProgressChanged;
 
         /// <summary>
         /// Registers the action.
@@ -21,22 +22,6 @@ namespace MediaBrowser.Common.Progress
         public void RegisterAction(Action<T> action)
         {
             _actions.Add(action);
-
-            ProgressChanged -= ActionableProgress_ProgressChanged;
-            ProgressChanged += ActionableProgress_ProgressChanged;
-        }
-
-        /// <summary>
-        /// Actionables the progress_ progress changed.
-        /// </summary>
-        /// <param name="sender">The sender.</param>
-        /// <param name="e">The e.</param>
-        void ActionableProgress_ProgressChanged(object sender, T e)
-        {
-            foreach (var action in _actions)
-            {
-                action(e);
-            }
         }
 
         /// <summary>
@@ -55,9 +40,34 @@ namespace MediaBrowser.Common.Progress
         {
             if (disposing)
             {
-                ProgressChanged -= ActionableProgress_ProgressChanged;
                 _actions.Clear();
             }
         }
+
+        public void Report(T value)
+        {
+            if (ProgressChanged != null)
+            {
+                ProgressChanged(this, value);
+            }
+
+            foreach (var action in _actions)
+            {
+                action(value);
+            }
+        }
+    }
+
+    public class SimpleProgress<T> : IProgress<T>
+    {
+        public event EventHandler<T> ProgressChanged;
+
+        public void Report(T value)
+        {
+            if (ProgressChanged != null)
+            {
+                ProgressChanged(this, value);
+            }
+        }
     }
 }

MediaBrowser.Controller/Channels/Channel.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using MediaBrowser.Model.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 
 namespace MediaBrowser.Controller.Channels
 {
@@ -51,7 +52,7 @@ namespace MediaBrowser.Controller.Channels
                     SortBy = query.SortBy,
                     SortOrder = query.SortOrder
 
-                }, new Progress<double>(), CancellationToken.None).Result;
+                }, new SimpleProgress<double>(), CancellationToken.None).Result;
             }
             catch
             {

MediaBrowser.Controller/Dto/DtoOptions.cs

@@ -10,7 +10,8 @@ namespace MediaBrowser.Controller.Dto
     {
         private static readonly List<ItemFields> DefaultExcludedFields = new List<ItemFields>
         {
-            ItemFields.SeasonUserData
+            ItemFields.SeasonUserData,
+            ItemFields.RefreshState
         };
 
         public List<ItemFields> Fields { get; set; }

MediaBrowser.Controller/Entities/Audio/MusicAlbum.cs

@@ -235,8 +235,6 @@ namespace MediaBrowser.Controller.Entities.Audio
             {
                 await RefreshArtists(refreshOptions, cancellationToken).ConfigureAwait(false);
             }
-
-            progress.Report(100);
         }
 
         private async Task RefreshArtists(MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken)

MediaBrowser.Controller/Entities/Audio/MusicArtist.cs

@@ -250,8 +250,6 @@ namespace MediaBrowser.Controller.Entities.Audio
                 percent /= totalItems;
                 progress.Report(percent * 100);
             }
-
-            progress.Report(100);
         }
 
         public ArtistInfo GetLookupInfo()

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -1058,6 +1058,16 @@ namespace MediaBrowser.Controller.Entities
             return RefreshMetadata(new MetadataRefreshOptions(new DirectoryService(Logger, FileSystem)), cancellationToken);
         }
 
+        protected virtual void TriggerOnRefreshStart()
+        {
+
+        }
+
+        protected virtual void TriggerOnRefreshComplete()
+        {
+
+        }
+
         /// <summary>
         /// Overrides the base implementation to refresh metadata for local trailers
         /// </summary>
@@ -1066,6 +1076,8 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>true if a provider reports we changed</returns>
         public async Task<ItemUpdateType> RefreshMetadata(MetadataRefreshOptions options, CancellationToken cancellationToken)
         {
+            TriggerOnRefreshStart();
+
             var locationType = LocationType;
 
             var requiresSave = false;
@@ -1091,6 +1103,8 @@ namespace MediaBrowser.Controller.Entities
                 }
             }
 
+            try
+            {
                 var refreshOptions = requiresSave
                     ? new MetadataRefreshOptions(options)
                     {
@@ -1100,6 +1114,11 @@ namespace MediaBrowser.Controller.Entities
 
                 return await ProviderManager.RefreshSingleItem(this, refreshOptions, cancellationToken).ConfigureAwait(false);
             }
+            finally
+            {
+                TriggerOnRefreshComplete();
+            }
+        }
 
         [IgnoreDataMember]
         protected virtual bool SupportsOwnedItems
@@ -2421,5 +2440,10 @@ namespace MediaBrowser.Controller.Entities
         {
             return new List<ExternalUrl>();
         }
+
+        public virtual double? GetRefreshProgress()
+        {
+            return null;
+        }
     }
 }

MediaBrowser.Controller/Entities/CollectionFolder.cs

@@ -10,6 +10,7 @@ using System.Threading.Tasks;
 
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Extensions;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Serialization;
@@ -199,6 +200,30 @@ namespace MediaBrowser.Controller.Entities
             return changed;
         }
 
+        public override double? GetRefreshProgress()
+        {
+            var folders = GetPhysicalFolders(true).ToList();
+            double totalProgresses = 0;
+            var foldersWithProgress = 0;
+
+            foreach (var folder in folders)
+            {
+                var progress = ProviderManager.GetRefreshProgress(folder.Id);
+                if (progress.HasValue)
+                {
+                    totalProgresses += progress.Value;
+                    foldersWithProgress++;
+                }
+            }
+
+            if (foldersWithProgress == 0)
+            {
+                return null;
+            }
+
+            return (totalProgresses / foldersWithProgress);
+        }
+
         protected override bool RefreshLinkedChildren(IEnumerable<FileSystemMetadata> fileSystemChildren)
         {
             return RefreshLinkedChildrenInternal(true);
@@ -321,6 +346,11 @@ namespace MediaBrowser.Controller.Entities
             return GetPhysicalFolders(true).SelectMany(c => c.Children);
         }
 
+        public IEnumerable<Folder> GetPhysicalFolders()
+        {
+            return GetPhysicalFolders(true);
+        }
+
         private IEnumerable<Folder> GetPhysicalFolders(bool enableCache)
         {
             if (enableCache)

MediaBrowser.Controller/Entities/Folder.cs

@@ -271,6 +271,11 @@ namespace MediaBrowser.Controller.Entities
             return GetCachedChildren();
         }
 
+        public override double? GetRefreshProgress()
+        {
+            return ProviderManager.GetRefreshProgress(Id);
+        }
+
         public Task ValidateChildren(IProgress<double> progress, CancellationToken cancellationToken)
         {
             return ValidateChildren(progress, cancellationToken, new MetadataRefreshOptions(new DirectoryService(Logger, FileSystem)));
@@ -318,6 +323,14 @@ namespace MediaBrowser.Controller.Entities
             return current.IsValidFromResolver(newItem);
         }
 
+        protected override void TriggerOnRefreshStart()
+        {
+        }
+
+        protected override void TriggerOnRefreshComplete()
+        {
+        }
+
         /// <summary>
         /// Validates the children internal.
         /// </summary>
@@ -328,7 +341,27 @@ namespace MediaBrowser.Controller.Entities
         /// <param name="refreshOptions">The refresh options.</param>
         /// <param name="directoryService">The directory service.</param>
         /// <returns>Task.</returns>
-        protected async virtual Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService)
+        protected virtual async Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService)
+        {
+            if (recursive)
+            {
+                ProviderManager.OnRefreshStart(this);
+            }
+
+            try
+            {
+                await ValidateChildrenInternal2(progress, cancellationToken, recursive, refreshChildMetadata, refreshOptions, directoryService).ConfigureAwait(false);
+            }
+            finally
+            {
+                if (recursive)
+                {
+                    ProviderManager.OnRefreshComplete(this);
+                }
+            }
+        }
+
+        private async Task ValidateChildrenInternal2(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService)
         {
             var locationType = LocationType;
 
@@ -360,6 +393,11 @@ namespace MediaBrowser.Controller.Entities
 
                 progress.Report(5);
 
+                if (recursive)
+                {
+                    ProviderManager.OnRefreshProgress(this, 5);
+                }
+
                 //build a dictionary of the current children we have now by Id so we can compare quickly and easily
                 var currentChildren = GetActualChildrenDictionary();
 
@@ -424,25 +462,63 @@ namespace MediaBrowser.Controller.Entities
                     await LibraryManager.CreateItems(newItems, cancellationToken).ConfigureAwait(false);
                 }
             }
+            else
+            {
+                if (recursive || refreshChildMetadata)
+                {
+                    // used below
+                    validChildren = Children.ToList();
+                }
+            }
 
             progress.Report(10);
 
+            if (recursive)
+            {
+                ProviderManager.OnRefreshProgress(this, 10);
+            }
+
             cancellationToken.ThrowIfCancellationRequested();
 
             if (recursive)
             {
-                await ValidateSubFolders(Children.ToList().OfType<Folder>().ToList(), directoryService, progress, cancellationToken).ConfigureAwait(false);
+                using (var innerProgress = new ActionableProgress<double>())
-            }
+                {
+                    var folder = this;
+                    innerProgress.RegisterAction(p =>
+                    {
+                        double newPct = .70 * p + 10;
+                        progress.Report(newPct);
+                        ProviderManager.OnRefreshProgress(folder, newPct);
+                    });
 
-            progress.Report(20);
+                    await ValidateSubFolders(validChildren.OfType<Folder>().ToList(), directoryService, innerProgress, cancellationToken).ConfigureAwait(false);
+                }
+            }
 
             if (refreshChildMetadata)
             {
+                progress.Report(80);
+
+                if (recursive)
+                {
+                    ProviderManager.OnRefreshProgress(this, 80);
+                }
+
                 var container = this as IMetadataContainer;
 
-                var innerProgress = new ActionableProgress<double>();
+                using (var innerProgress = new ActionableProgress<double>())
-
+                {
-                innerProgress.RegisterAction(p => progress.Report(.80 * p + 20));
+                    var folder = this;
+                    innerProgress.RegisterAction(p =>
+                    {
+                        double newPct = .20 * p + 80;
+                        progress.Report(newPct);
+                        if (recursive)
+                        {
+                            ProviderManager.OnRefreshProgress(folder, newPct);
+                        }
+                    });
 
                     if (container != null)
                     {
@@ -450,50 +526,35 @@ namespace MediaBrowser.Controller.Entities
                     }
                     else
                     {
-                    await RefreshMetadataRecursive(refreshOptions, recursive, innerProgress, cancellationToken);
+                        await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken);
+                    }
+                }
             }
         }
 
-            progress.Report(100);
+        private async Task RefreshMetadataRecursive(List<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
-        }
-
-        private async Task RefreshMetadataRecursive(MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var children = Children.ToList();
-
-            var percentages = new Dictionary<Guid, double>(children.Count);
             var numComplete = 0;
             var count = children.Count;
+            double currentPercent = 0;
 
             foreach (var child in children)
             {
                 cancellationToken.ThrowIfCancellationRequested();
 
-                if (child.IsFolder)
+                using (var innerProgress = new ActionableProgress<double>())
                 {
-                    var innerProgress = new ActionableProgress<double>();
-
                     // Avoid implicitly captured closure
-                    var currentChild = child;
+                    var currentInnerPercent = currentPercent;
+
                     innerProgress.RegisterAction(p =>
                     {
-                        lock (percentages)
+                        double innerPercent = currentInnerPercent;
-                        {
+                        innerPercent += p / (count);
-                            percentages[currentChild.Id] = p / 100;
-
-                            var innerPercent = percentages.Values.Sum();
-                            innerPercent /= count;
-                            innerPercent *= 100;
                         progress.Report(innerPercent);
-                        }
                     });
 
-                    await RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken)
+                    await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken)
-                      .ConfigureAwait(false);
-                }
-                else
-                {
-                    await RefreshChildMetadata(child, refreshOptions, false, new Progress<double>(), cancellationToken)
                       .ConfigureAwait(false);
                 }
 
@@ -501,11 +562,10 @@ namespace MediaBrowser.Controller.Entities
                 double percent = numComplete;
                 percent /= count;
                 percent *= 100;
+                currentPercent = percent;
 
                 progress.Report(percent);
             }
-
-            progress.Report(100);
         }
 
         private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
@@ -526,11 +586,10 @@ namespace MediaBrowser.Controller.Entities
 
                     if (folder != null)
                     {
-                        await folder.RefreshMetadataRecursive(refreshOptions, true, progress, cancellationToken);
+                        await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken);
                     }
                 }
             }
-            progress.Report(100);
         }
 
         /// <summary>
@@ -543,45 +602,38 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>Task.</returns>
         private async Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var list = children;
+            var numComplete = 0;
-            var childCount = list.Count;
+            var count = children.Count;
+            double currentPercent = 0;
 
-            var percentages = new Dictionary<Guid, double>(list.Count);
+            foreach (var child in children)
-
-            foreach (var item in list)
             {
                 cancellationToken.ThrowIfCancellationRequested();
 
-                var child = item;
+                using (var innerProgress = new ActionableProgress<double>())
-
+                {
-                var innerProgress = new ActionableProgress<double>();
+                    // Avoid implicitly captured closure
+                    var currentInnerPercent = currentPercent;
 
                     innerProgress.RegisterAction(p =>
                     {
-                    lock (percentages)
+                        double innerPercent = currentInnerPercent;
-                    {
+                        innerPercent += p / (count);
-                        percentages[child.Id] = p / 100;
+                        progress.Report(innerPercent);
-
-                        var percent = percentages.Values.Sum();
-                        percent /= childCount;
-
-                        progress.Report(10 * percent + 10);
-                    }
                     });
 
                     await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService)
                             .ConfigureAwait(false);
                 }
-        }
 
-        /// <summary>
+                numComplete++;
-        /// Determines whether the specified path is offline.
+                double percent = numComplete;
-        /// </summary>
+                percent /= count;
-        /// <param name="path">The path.</param>
+                percent *= 100;
-        /// <returns><c>true</c> if the specified path is offline; otherwise, <c>false</c>.</returns>
+                currentPercent = percent;
-        public static bool IsPathOffline(string path)
+
-        {
+                progress.Report(percent);
-            return IsPathOffline(path, LibraryManager.GetVirtualFolders().SelectMany(i => i.Locations).ToList());
+            }
         }
 
         public static bool IsPathOffline(string path, List<string> allLibraryPaths)
@@ -926,7 +978,7 @@ namespace MediaBrowser.Controller.Entities
                         SortBy = query.SortBy,
                         SortOrder = query.SortOrder
 
-                    }, new Progress<double>(), CancellationToken.None).Result;
+                    }, new SimpleProgress<double>(), CancellationToken.None).Result;
                 }
                 catch
                 {

MediaBrowser.Controller/Entities/TV/Series.cs

@@ -414,8 +414,6 @@ namespace MediaBrowser.Controller.Entities.TV
             refreshOptions = new MetadataRefreshOptions(refreshOptions);
             refreshOptions.IsPostRecursiveRefresh = true;
             await ProviderManager.RefreshSingleItem(this, refreshOptions, cancellationToken).ConfigureAwait(false);
-
-            progress.Report(100);
         }
 
         public IEnumerable<Episode> GetSeasonEpisodes(Season parentSeason, User user, DtoOptions options)

MediaBrowser.Controller/Library/ILibraryManager.cs

@@ -132,7 +132,9 @@ namespace MediaBrowser.Controller.Library
         /// Gets the default view.
         /// </summary>
         /// <returns>IEnumerable{VirtualFolderInfo}.</returns>
-        IEnumerable<VirtualFolderInfo> GetVirtualFolders();
+        List<VirtualFolderInfo> GetVirtualFolders();
+
+        List<VirtualFolderInfo> GetVirtualFolders(bool includeRefreshState);
 
         /// <summary>
         /// Gets the item by id.

MediaBrowser.Controller/MediaBrowser.Controller.csproj

@@ -320,7 +320,6 @@
     <Compile Include="Plugins\IPluginConfigurationPage.cs" />
     <Compile Include="Plugins\IServerEntryPoint.cs" />
     <Compile Include="Providers\IImageEnhancer.cs" />
-    <Compile Include="Providers\ProviderRefreshStatus.cs" />
     <Compile Include="Resolvers\IResolverIgnoreRule.cs" />
     <Compile Include="Resolvers\ResolverPriority.cs" />
     <Compile Include="Library\TVUtils.cs" />

MediaBrowser.Controller/Providers/IProviderManager.cs

@@ -9,6 +9,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Model.Events;
 
 namespace MediaBrowser.Controller.Providers
 {
@@ -158,6 +159,18 @@ namespace MediaBrowser.Controller.Providers
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task{HttpResponseInfo}.</returns>
         Task<HttpResponseInfo> GetSearchImage(string providerName, string url, CancellationToken cancellationToken);
+
+        Dictionary<Guid, Guid> GetRefreshQueue();
+
+        void OnRefreshStart(BaseItem item);
+        void OnRefreshProgress(BaseItem item, double progress);
+        void OnRefreshComplete(BaseItem item);
+
+        double? GetRefreshProgress(Guid id);
+
+        event EventHandler<GenericEventArgs<BaseItem>> RefreshStarted;
+        event EventHandler<GenericEventArgs<BaseItem>> RefreshCompleted;
+        event EventHandler<GenericEventArgs<Tuple<BaseItem, double>>> RefreshProgress;
     }
 
     public enum RefreshPriority

MediaBrowser.Controller/Providers/ProviderRefreshStatus.cs

@@ -1,22 +0,0 @@
-
-namespace MediaBrowser.Controller.Providers
-{
-    /// <summary>
-    /// Enum ProviderRefreshStatus
-    /// </summary>
-    public enum ProviderRefreshStatus
-    {
-        /// <summary>
-        /// The success
-        /// </summary>
-        Success = 0,
-        /// <summary>
-        /// The completed with errors
-        /// </summary>
-        CompletedWithErrors = 1,
-         /// <summary>
-        /// The failure
-        /// </summary>
-        Failure = 2
-   }
-}

MediaBrowser.MediaEncoding/Encoder/FontConfigLoader.cs

@@ -7,6 +7,7 @@ using MediaBrowser.Model.IO;
 using MediaBrowser.Common.Configuration;
 
 using MediaBrowser.Common.Net;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Logging;
@@ -62,7 +63,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     // Kick this off, but no need to wait on it
                     var task = Task.Run(async () =>
                     {
-                        await DownloadFontFile(fontsDirectory, fontFilename, new Progress<double>()).ConfigureAwait(false);
+                        await DownloadFontFile(fontsDirectory, fontFilename, new SimpleProgress<double>()).ConfigureAwait(false);
 
                         await WriteFontConfigFile(fontsDirectory).ConfigureAwait(false);
                     });

MediaBrowser.MediaEncoding/Subtitles/OpenSubtitleDownloader.cs

@@ -136,7 +136,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
             if ((DateTime.UtcNow - _lastRateLimitException).TotalHours < 1)
             {
-                throw new Exception("OpenSubtitles rate limit reached");
+                throw new RateLimitExceededException("OpenSubtitles rate limit reached");
             }
 
             var resultDownLoad = await OpenSubtitles.DownloadSubtitlesAsync(downloadsList, cancellationToken).ConfigureAwait(false);
@@ -144,7 +144,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             if ((resultDownLoad.Status ?? string.Empty).IndexOf("407", StringComparison.OrdinalIgnoreCase) != -1)
             {
                 _lastRateLimitException = DateTime.UtcNow;
-                throw new Exception("OpenSubtitles rate limit reached");
+                throw new RateLimitExceededException("OpenSubtitles rate limit reached");
             }
 
             if (!(resultDownLoad is MethodResponseSubtitleDownload))

MediaBrowser.Model/Entities/VirtualFolderInfo.cs

@@ -47,5 +47,8 @@ namespace MediaBrowser.Model.Entities
         /// </summary>
         /// <value>The primary image item identifier.</value>
         public string PrimaryImageItemId { get; set; }
+
+        public double? RefreshProgress { get; set; }
+        public string RefreshStatus { get; set; }
     }
 }

MediaBrowser.Model/Querying/ItemFields.cs

@@ -228,6 +228,7 @@
         ExternalSeriesId,
         SeriesPresentationUniqueKey,
         DateLastRefreshed,
-        DateLastSaved
+        DateLastSaved,
+        RefreshState
     }
 }

MediaBrowser.Providers/ImagesByName/ImageUtils.cs

@@ -6,6 +6,7 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.IO;
 
 namespace MediaBrowser.Providers.ImagesByName
@@ -30,7 +31,7 @@ namespace MediaBrowser.Providers.ImagesByName
                 var temp = await httpClient.GetTempFile(new HttpRequestOptions
                 {
                     CancellationToken = cancellationToken,
-                    Progress = new Progress<double>(),
+                    Progress = new SimpleProgress<double>(),
                     Url = url
 
                 }).ConfigureAwait(false);

MediaBrowser.Providers/Manager/ProviderManager.cs

@@ -18,8 +18,10 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Controller.Dto;
+using MediaBrowser.Model.Events;
 using MediaBrowser.Model.Serialization;
 using Priority_Queue;
 
@@ -67,6 +69,10 @@ namespace MediaBrowser.Providers.Manager
         private readonly IMemoryStreamFactory _memoryStreamProvider;
         private CancellationTokenSource _disposeCancellationTokenSource = new CancellationTokenSource();
 
+        public event EventHandler<GenericEventArgs<BaseItem>> RefreshStarted;
+        public event EventHandler<GenericEventArgs<BaseItem>> RefreshCompleted;
+        public event EventHandler<GenericEventArgs<Tuple<BaseItem, double>>> RefreshProgress;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="ProviderManager" /> class.
         /// </summary>
@@ -849,6 +855,89 @@ namespace MediaBrowser.Providers.Manager
                 });
         }
 
+        private Dictionary<Guid, double> _activeRefreshes = new Dictionary<Guid, double>();
+
+        public Dictionary<Guid, Guid> GetRefreshQueue()
+        {
+            lock (_refreshQueueLock)
+            {
+                var dict = new Dictionary<Guid, Guid>();
+
+                foreach (var item in _refreshQueue)
+                {
+                    dict[item.Item1] = item.Item1;
+                }
+                return dict;
+            }
+        }
+
+        public void OnRefreshStart(BaseItem item)
+        {
+            //_logger.Info("OnRefreshStart {0}", item.Id.ToString("N"));
+            var id = item.Id;
+
+            lock (_activeRefreshes)
+            {
+                _activeRefreshes[id] = 0;
+            }
+
+            if (RefreshStarted != null)
+            {
+                RefreshStarted(this, new GenericEventArgs<BaseItem>(item));
+            }
+        }
+
+        public void OnRefreshComplete(BaseItem item)
+        {
+            //_logger.Info("OnRefreshComplete {0}", item.Id.ToString("N"));
+            lock (_activeRefreshes)
+            {
+                _activeRefreshes.Remove(item.Id);
+            }
+
+            if (RefreshCompleted != null)
+            {
+                RefreshCompleted(this, new GenericEventArgs<BaseItem>(item));
+            }
+        }
+
+        public double? GetRefreshProgress(Guid id)
+        {
+            lock (_activeRefreshes)
+            {
+                double value;
+                if (_activeRefreshes.TryGetValue(id, out value))
+                {
+                    return value;
+                }
+
+                return null;
+            }
+        }
+
+        public void OnRefreshProgress(BaseItem item, double progress)
+        {
+            //_logger.Info("OnRefreshProgress {0} {1}", item.Id.ToString("N"), progress);
+            var id = item.Id;
+
+            lock (_activeRefreshes)
+            {
+                if (_activeRefreshes.ContainsKey(id))
+                {
+                    _activeRefreshes[id] = progress;
+
+                    if (RefreshProgress != null)
+                    {
+                        RefreshProgress(this, new GenericEventArgs<Tuple<BaseItem, double>>(new Tuple<BaseItem, double>(item, progress)));
+                    }
+                }
+                else
+                {
+                    throw new Exception(string.Format("Refresh for item {0} {1} is not in progress", item.GetType().Name, item.Id.ToString("N")));
+                }
+            }
+        }
+
         private readonly SimplePriorityQueue<Tuple<Guid, MetadataRefreshOptions>> _refreshQueue =
             new SimplePriorityQueue<Tuple<Guid, MetadataRefreshOptions>>();
 
@@ -906,7 +995,7 @@ namespace MediaBrowser.Providers.Manager
                             var folder = item as Folder;
                             if (folder != null)
                             {
-                                await folder.ValidateChildren(new Progress<double>(), cancellationToken).ConfigureAwait(false);
+                                await folder.ValidateChildren(new SimpleProgress<double>(), cancellationToken).ConfigureAwait(false);
                             }
                         }
 
@@ -951,14 +1040,14 @@ namespace MediaBrowser.Providers.Manager
 
                 if (folder != null)
                 {
-                    await folder.ValidateChildren(new Progress<double>(), cancellationToken, options).ConfigureAwait(false);
+                    await folder.ValidateChildren(new SimpleProgress<double>(), cancellationToken, options).ConfigureAwait(false);
                 }
             }
         }
 
         private async Task RefreshCollectionFolderChildren(MetadataRefreshOptions options, CollectionFolder collectionFolder, CancellationToken cancellationToken)
         {
-            foreach (var child in collectionFolder.Children.ToList())
+            foreach (var child in collectionFolder.GetPhysicalFolders().ToList())
             {
                 await child.RefreshMetadata(options, cancellationToken).ConfigureAwait(false);
 
@@ -966,7 +1055,7 @@ namespace MediaBrowser.Providers.Manager
                 {
                     var folder = (Folder)child;
 
-                    await folder.ValidateChildren(new Progress<double>(), cancellationToken, options, true).ConfigureAwait(false);
+                    await folder.ValidateChildren(new SimpleProgress<double>(), cancellationToken, options, true).ConfigureAwait(false);
                 }
             }
         }
@@ -991,7 +1080,7 @@ namespace MediaBrowser.Providers.Manager
                 .Where(i => i != null)
                 .ToList();
 
-            var musicArtistRefreshTasks = musicArtists.Select(i => i.ValidateChildren(new Progress<double>(), cancellationToken, options, true));
+            var musicArtistRefreshTasks = musicArtists.Select(i => i.ValidateChildren(new SimpleProgress<double>(), cancellationToken, options, true));
 
             await Task.WhenAll(musicArtistRefreshTasks).ConfigureAwait(false);
 

MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs

@@ -10,6 +10,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Common.Extensions;
 
 namespace MediaBrowser.Providers.MediaInfo
 {
@@ -154,6 +155,9 @@ namespace MediaBrowser.Providers.MediaInfo
                     return true;
                 }
             }
+            catch (RateLimitExceededException)
+            {
+            }
             catch (Exception ex)
             {
                 _logger.ErrorException("Error downloading subtitles", ex);

MediaBrowser.Providers/Subtitles/SubtitleManager.cs

@@ -133,7 +133,8 @@ namespace MediaBrowser.Providers.Subtitles
                     {
                         //var isText = MediaStream.IsTextFormat(response.Format);
 
-                        using (var fs = _fileSystem.GetFileStream(savePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true))
+                        using (var fs = _fileSystem.GetFileStream(savePath, FileOpenMode.Create, FileAccessMode.Write,
+                                FileShareMode.Read, true))
                         {
                             await stream.CopyToAsync(fs).ConfigureAwait(false);
                         }
@@ -154,6 +155,10 @@ namespace MediaBrowser.Providers.Subtitles
                     }
                 }
             }
+            catch (RateLimitExceededException)
+            {
+                throw;
+            }
             catch (Exception ex)
             {
                 EventHelper.FireEventIfNotNull(SubtitleDownloadFailure, this, new SubtitleDownloadFailureEventArgs

MediaBrowser.Providers/TV/DummySeasonProvider.cs

@@ -46,7 +46,7 @@ namespace MediaBrowser.Providers.TV
 
                 //await series.RefreshMetadata(new MetadataRefreshOptions(directoryService), cancellationToken).ConfigureAwait(false);
 
-                //await series.ValidateChildren(new Progress<double>(), cancellationToken, new MetadataRefreshOptions(directoryService))
+                //await series.ValidateChildren(new SimpleProgress<double>(), cancellationToken, new MetadataRefreshOptions(directoryService))
                 //    .ConfigureAwait(false);
             }
         }

MediaBrowser.Providers/TV/MissingEpisodeProvider.cs

@@ -13,7 +13,7 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
-
+using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Globalization;
@@ -162,7 +162,7 @@ namespace MediaBrowser.Providers.TV
 
                     await series.RefreshMetadata(new MetadataRefreshOptions(directoryService), cancellationToken).ConfigureAwait(false);
 
-                    await series.ValidateChildren(new Progress<double>(), cancellationToken, new MetadataRefreshOptions(directoryService), true)
+                    await series.ValidateChildren(new SimpleProgress<double>(), cancellationToken, new MetadataRefreshOptions(directoryService), true)
                         .ConfigureAwait(false);
                 }
             }

SharedVersion.cs

@@ -1,3 +1,3 @@
 using System.Reflection;
 
-[assembly: AssemblyVersion("3.2.20.10")]
+[assembly: AssemblyVersion("3.2.20.11")]