From 04d480297dc9b4f9a1a9e35fe33f424ecc7662e1 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 3 Aug 2016 02:38:19 -0400 Subject: [PATCH] reduce sync db access --- .../UserLibrary/BaseItemsByNameService.cs | 6 +- MediaBrowser.Controller/Dto/IDtoService.cs | 11 +- MediaBrowser.Controller/Sync/ISyncManager.cs | 2 +- .../Sync/ISyncRepository.cs | 5 +- .../Sync/SyncedItemProgress.cs | 2 +- MediaBrowser.Model/Dto/IHasSyncInfo.cs | 1 + .../Dto/DtoService.cs | 113 ++++++++---------- .../Sync/SyncManager.cs | 2 +- .../Sync/SyncRepository.cs | 99 ++++++++++++--- 9 files changed, 145 insertions(+), 96 deletions(-) diff --git a/MediaBrowser.Api/UserLibrary/BaseItemsByNameService.cs b/MediaBrowser.Api/UserLibrary/BaseItemsByNameService.cs index 9465d1fdc..852a3d59e 100644 --- a/MediaBrowser.Api/UserLibrary/BaseItemsByNameService.cs +++ b/MediaBrowser.Api/UserLibrary/BaseItemsByNameService.cs @@ -180,9 +180,10 @@ namespace MediaBrowser.Api.UserLibrary var result = GetItems(request, query); + var syncProgess = DtoService.GetSyncedItemProgress(dtoOptions); var dtos = result.Items.Select(i => { - var dto = DtoService.GetItemByNameDto(i.Item1, dtoOptions, null, user); + var dto = DtoService.GetItemByNameDto(i.Item1, dtoOptions, null, syncProgess, user); if (!string.IsNullOrWhiteSpace(request.IncludeItemTypes)) { @@ -325,7 +326,8 @@ namespace MediaBrowser.Api.UserLibrary tuples = ibnItems.Select(i => new Tuple>(i, new List())); } - var dtos = tuples.Select(i => DtoService.GetItemByNameDto(i.Item1, dtoOptions, i.Item2, user)); + var syncProgess = DtoService.GetSyncedItemProgress(dtoOptions); + var dtos = tuples.Select(i => DtoService.GetItemByNameDto(i.Item1, dtoOptions, i.Item2, syncProgess, user)); result.Items = dtos.Where(i => i != null).ToArray(); diff --git a/MediaBrowser.Controller/Dto/IDtoService.cs b/MediaBrowser.Controller/Dto/IDtoService.cs index e4aa466df..a8e93fb2b 100644 --- a/MediaBrowser.Controller/Dto/IDtoService.cs +++ b/MediaBrowser.Controller/Dto/IDtoService.cs @@ -4,6 +4,8 @@ using MediaBrowser.Model.Dto; using MediaBrowser.Model.Querying; using System.Collections.Generic; using System.Threading.Tasks; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Sync; namespace MediaBrowser.Controller.Dto { @@ -89,11 +91,8 @@ namespace MediaBrowser.Controller.Dto /// /// Gets the item by name dto. /// - /// The item. - /// The options. - /// The tagged items. - /// The user. - /// BaseItemDto. - BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, User user = null); + BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, Dictionary syncProgress, User user = null); + + Dictionary GetSyncedItemProgress(DtoOptions options); } } diff --git a/MediaBrowser.Controller/Sync/ISyncManager.cs b/MediaBrowser.Controller/Sync/ISyncManager.cs index 8eb934eaa..fbcb549ad 100644 --- a/MediaBrowser.Controller/Sync/ISyncManager.cs +++ b/MediaBrowser.Controller/Sync/ISyncManager.cs @@ -151,7 +151,7 @@ namespace MediaBrowser.Controller.Sync /// /// The query. /// QueryResult<System.String>. - QueryResult GetSyncedItemProgresses(SyncJobItemQuery query); + Dictionary GetSyncedItemProgresses(SyncJobItemQuery query); /// /// Reports the synchronize job item transfer beginning. diff --git a/MediaBrowser.Controller/Sync/ISyncRepository.cs b/MediaBrowser.Controller/Sync/ISyncRepository.cs index 2af09dbaa..8e9b2bf77 100644 --- a/MediaBrowser.Controller/Sync/ISyncRepository.cs +++ b/MediaBrowser.Controller/Sync/ISyncRepository.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Model.Querying; +using System.Collections.Generic; +using MediaBrowser.Model.Querying; using MediaBrowser.Model.Sync; using System.Threading.Tasks; @@ -74,6 +75,6 @@ namespace MediaBrowser.Controller.Sync /// /// The query. /// QueryResult<System.String>. - QueryResult GetSyncedItemProgresses(SyncJobItemQuery query); + Dictionary GetSyncedItemProgresses(SyncJobItemQuery query); } } diff --git a/MediaBrowser.Controller/Sync/SyncedItemProgress.cs b/MediaBrowser.Controller/Sync/SyncedItemProgress.cs index edb42eb0f..0fd929eb1 100644 --- a/MediaBrowser.Controller/Sync/SyncedItemProgress.cs +++ b/MediaBrowser.Controller/Sync/SyncedItemProgress.cs @@ -4,7 +4,7 @@ namespace MediaBrowser.Controller.Sync { public class SyncedItemProgress { - public string ItemId { get; set; } + public double Progress { get; set; } public SyncJobItemStatus Status { get; set; } } } diff --git a/MediaBrowser.Model/Dto/IHasSyncInfo.cs b/MediaBrowser.Model/Dto/IHasSyncInfo.cs index d2cf1f8cf..7471dacdd 100644 --- a/MediaBrowser.Model/Dto/IHasSyncInfo.cs +++ b/MediaBrowser.Model/Dto/IHasSyncInfo.cs @@ -7,6 +7,7 @@ namespace MediaBrowser.Model.Dto string Id { get; } bool? SupportsSync { get; set; } bool? HasSyncJob { get; set; } + double? SyncPercent { get; set; } bool? IsSynced { get; set; } SyncJobItemStatus? SyncStatus { get; set; } } diff --git a/MediaBrowser.Server.Implementations/Dto/DtoService.cs b/MediaBrowser.Server.Implementations/Dto/DtoService.cs index 90111ad8e..779afdcf2 100644 --- a/MediaBrowser.Server.Implementations/Dto/DtoService.cs +++ b/MediaBrowser.Server.Implementations/Dto/DtoService.cs @@ -97,8 +97,7 @@ namespace MediaBrowser.Server.Implementations.Dto throw new ArgumentNullException("options"); } - var syncJobItems = GetSyncedItemProgress(options); - var syncDictionary = GetSyncedItemProgressDictionary(syncJobItems); + var syncDictionary = GetSyncedItemProgress(options); var list = new List(); var programTuples = new List>(); @@ -106,7 +105,7 @@ namespace MediaBrowser.Server.Implementations.Dto foreach (var item in items) { - var dto = await GetBaseItemDtoInternal(item, options, syncDictionary, user, owner).ConfigureAwait(false); + var dto = await GetBaseItemDtoInternal(item, options, user, owner).ConfigureAwait(false); var tvChannel = item as LiveTvChannel; if (tvChannel != null) @@ -133,7 +132,7 @@ namespace MediaBrowser.Server.Implementations.Dto } } - FillSyncInfo(dto, item, options, user, syncJobItems); + FillSyncInfo(dto, item, options, user, syncDictionary); list.Add(dto); } @@ -151,23 +150,11 @@ namespace MediaBrowser.Server.Implementations.Dto return list; } - private Dictionary GetSyncedItemProgressDictionary(IEnumerable items) - { - var dict = new Dictionary(); - - foreach (var item in items) - { - dict[item.ItemId] = item; - } - - return dict; - } - public BaseItemDto GetBaseItemDto(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null) { - var syncProgress = GetSyncedItemProgress(options); + var syncDictionary = GetSyncedItemProgress(options); - var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user, owner).Result; + var dto = GetBaseItemDtoInternal(item, options, user, owner).Result; var tvChannel = item as LiveTvChannel; if (tvChannel != null) { @@ -190,11 +177,11 @@ namespace MediaBrowser.Server.Implementations.Dto SetItemByNameInfo(item, dto, GetTaggedItems(byName, user), user); } - FillSyncInfo(dto, item, options, user, syncProgress); + FillSyncInfo(dto, item, options, user, syncDictionary); return dto; } - FillSyncInfo(dto, item, options, user, syncProgress); + FillSyncInfo(dto, item, options, user, syncDictionary); return dto; } @@ -210,23 +197,24 @@ namespace MediaBrowser.Server.Implementations.Dto return items; } - private SyncedItemProgress[] GetSyncedItemProgress(DtoOptions options) + public Dictionary GetSyncedItemProgress(DtoOptions options) { - if (!options.Fields.Contains(ItemFields.SyncInfo)) + if (!options.Fields.Contains(ItemFields.BasicSyncInfo) && + !options.Fields.Contains(ItemFields.SyncInfo)) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } var deviceId = options.DeviceId; if (string.IsNullOrWhiteSpace(deviceId)) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } var caps = _deviceManager().GetCapabilities(deviceId); if (caps == null || !caps.SupportsSync) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } return _syncManager.GetSyncedItemProgresses(new SyncJobItemQuery @@ -240,12 +228,13 @@ namespace MediaBrowser.Server.Implementations.Dto SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced } - }).Items; + }); } public void FillSyncInfo(IEnumerable> tuples, DtoOptions options, User user) { - if (options.Fields.Contains(ItemFields.SyncInfo)) + if (options.Fields.Contains(ItemFields.BasicSyncInfo) || + options.Fields.Contains(ItemFields.SyncInfo)) { var syncProgress = GetSyncedItemProgress(options); @@ -258,37 +247,40 @@ namespace MediaBrowser.Server.Implementations.Dto } } - private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, SyncedItemProgress[] syncProgress) + private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, Dictionary syncProgress) { var hasFullSyncInfo = options.Fields.Contains(ItemFields.SyncInfo); - if (hasFullSyncInfo || options.Fields.Contains(ItemFields.BasicSyncInfo)) + if (!options.Fields.Contains(ItemFields.BasicSyncInfo) && + !hasFullSyncInfo) { - var userCanSync = user != null && user.Policy.EnableSync; - dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item); + return; } if (dto.SupportsSync ?? false) { - dto.HasSyncJob = syncProgress.Any(i => i.Status != SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); - dto.IsSynced = syncProgress.Any(i => i.Status == SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); - - if (hasFullSyncInfo) + SyncedItemProgress syncStatus; + if (syncProgress.TryGetValue(dto.Id, out syncStatus)) { - if (dto.IsSynced.Value) + if (syncStatus.Status == SyncJobItemStatus.Synced) { - dto.SyncStatus = SyncJobItemStatus.Synced; + dto.SyncPercent = 100; + } + else + { + dto.SyncPercent = syncStatus.Progress; } - else if (dto.HasSyncJob.Value) + if (hasFullSyncInfo) { - dto.SyncStatus = syncProgress.Where(i => string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)).Select(i => i.Status).Max(); + dto.HasSyncJob = true; + dto.SyncStatus = syncStatus.Status; } } } } - private async Task GetBaseItemDtoInternal(BaseItem item, DtoOptions options, Dictionary syncProgress, User user = null, BaseItem owner = null) + private async Task GetBaseItemDtoInternal(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null) { var fields = options.Fields; @@ -337,7 +329,7 @@ namespace MediaBrowser.Server.Implementations.Dto if (user != null) { - await AttachUserSpecificInfo(dto, item, user, fields, syncProgress).ConfigureAwait(false); + await AttachUserSpecificInfo(dto, item, user, fields).ConfigureAwait(false); } var hasMediaSources = item as IHasMediaSources; @@ -400,11 +392,9 @@ namespace MediaBrowser.Server.Implementations.Dto return dto; } - public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, User user = null) + public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, Dictionary syncProgress, User user = null) { - var syncProgress = GetSyncedItemProgress(options); - - var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user).Result; + var dto = GetBaseItemDtoInternal(item, options, user).Result; if (taggedItems != null && options.Fields.Contains(ItemFields.ItemCounts)) { @@ -452,28 +442,13 @@ namespace MediaBrowser.Server.Implementations.Dto /// The item. /// The user. /// The fields. - /// The synchronize progress. - private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List fields, Dictionary syncProgress) + private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List fields) { if (item.IsFolder) { var folder = (Folder)item; - if (item.SourceType == SourceType.Library && folder.SupportsUserDataFromChildren && fields.Contains(ItemFields.SyncInfo)) - { - // Skip the user data manager because we've already looped through the recursive tree and don't want to do it twice - // TODO: Improve in future - dto.UserData = GetUserItemDataDto(_userDataRepository.GetUserData(user, item)); - - await SetSpecialCounts(folder, user, dto, fields, syncProgress).ConfigureAwait(false); - - dto.UserData.Played = dto.UserData.PlayedPercentage.HasValue && - dto.UserData.PlayedPercentage.Value >= 100; - } - else - { - dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false); - } + dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false); if (item.SourceType == SourceType.Library) { @@ -498,6 +473,12 @@ namespace MediaBrowser.Server.Implementations.Dto dto.PlayAccess = item.GetPlayAccess(user); + if (fields.Contains(ItemFields.BasicSyncInfo) || fields.Contains(ItemFields.SyncInfo)) + { + var userCanSync = user != null && user.Policy.EnableSync; + dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item); + } + if (fields.Contains(ItemFields.SeasonUserData)) { var episode = item as Episode; @@ -1424,7 +1405,7 @@ namespace MediaBrowser.Server.Implementations.Dto BaseItem parent = null; var isFirst = true; - while (((!dto.HasLogo && logoLimit > 0) || (!dto.HasArtImage && artLimit > 0) || (!dto.HasThumb && thumbLimit > 0) || parent is Series) && + while (((!dto.HasLogo && logoLimit > 0) || (!dto.HasArtImage && artLimit > 0) || (!dto.HasThumb && thumbLimit > 0) || parent is Series) && (parent = parent ?? (isFirst ? item.GetParent() ?? owner : parent)) != null) { if (parent == null) @@ -1531,7 +1512,7 @@ namespace MediaBrowser.Server.Implementations.Dto /// The fields. /// The synchronize progress. /// Task. - private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List fields, Dictionary syncProgress) + private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List fields, Dictionary syncProgress) { var recursiveItemCount = 0; var unplayed = 0; @@ -1580,10 +1561,10 @@ namespace MediaBrowser.Server.Implementations.Dto } double percent = 0; - SyncedItemProgress syncItemProgress; + SyncJobItemStatus syncItemProgress; if (syncProgress.TryGetValue(child.Id.ToString("N"), out syncItemProgress)) { - switch (syncItemProgress.Status) + switch (syncItemProgress) { case SyncJobItemStatus.Synced: percent = 100; diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs index 15855f553..ffba60af8 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs @@ -1124,7 +1124,7 @@ namespace MediaBrowser.Server.Implementations.Sync await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); } - public QueryResult GetSyncedItemProgresses(SyncJobItemQuery query) + public Dictionary GetSyncedItemProgresses(SyncJobItemQuery query) { return _repo.GetSyncedItemProgresses(query); } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs index a1ed66a99..df4149fa7 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs @@ -29,6 +29,17 @@ namespace MediaBrowser.Server.Implementations.Sync DbFilePath = Path.Combine(appPaths.DataPath, "sync14.db"); } + private class SyncSummary + { + public Dictionary Items { get; set; } + + public SyncSummary() + { + Items = new Dictionary(); + } + } + + public async Task Initialize() { using (var connection = await CreateConnection().ConfigureAwait(false)) @@ -41,7 +52,10 @@ namespace MediaBrowser.Server.Implementations.Sync "create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, TemporaryPath TEXT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT, AdditionalFiles TEXT, MediaSource TEXT, IsMarkedForRemoval BIT, JobItemIndex INT, ItemDateModifiedTicks BIGINT)", "create index if not exists idx_SyncJobItems1 on SyncJobItems(Id)", - "create index if not exists idx_SyncJobItems2 on SyncJobItems(TargetId)", + + "drop index if exists idx_SyncJobItems2", + "drop index if exists idx_SyncJobItems3", + "create index if not exists idx_SyncJobItems4 on SyncJobItems(TargetId,ItemId,Status,Progress,DateCreated)", "pragma shrink_memory" }; @@ -583,9 +597,74 @@ namespace MediaBrowser.Server.Implementations.Sync } } - public QueryResult GetSyncedItemProgresses(SyncJobItemQuery query) + public Dictionary GetSyncedItemProgresses(SyncJobItemQuery query) { - return GetJobItemReader(query, "select ItemId,Status from SyncJobItems", GetSyncedItemProgress); + var result = new Dictionary(); + + using (var connection = CreateConnection(true).Result) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "select ItemId,Progress,Status from SyncJobItems"; + + var whereClauses = new List(); + + if (!string.IsNullOrWhiteSpace(query.TargetId)) + { + whereClauses.Add("TargetId=@TargetId"); + cmd.Parameters.Add(cmd, "@TargetId", DbType.String).Value = query.TargetId; + } + + if (query.Statuses.Length > 0) + { + var statuses = string.Join(",", query.Statuses.Select(i => "'" + i.ToString() + "'").ToArray()); + + whereClauses.Add(string.Format("Status in ({0})", statuses)); + } + + if (whereClauses.Count > 0) + { + cmd.CommandText += " where " + string.Join(" AND ", whereClauses.ToArray()); + } + + using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult)) + { + while (reader.Read()) + { + var itemId = reader.GetString(0); + + if (!reader.IsDBNull(1)) + { + var status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true); + + if (status == SyncJobItemStatus.Synced) + { + result[itemId] = new SyncedItemProgress + { + Status = SyncJobItemStatus.Synced + }; + } + else + { + SyncedItemProgress currentStatus; + double progress = reader.IsDBNull(2) ? 0.0 : reader.GetDouble(2); + + if (!result.TryGetValue(itemId, out currentStatus) || (currentStatus.Status != SyncJobItemStatus.Synced && progress >= currentStatus.Progress)) + { + result[itemId] = new SyncedItemProgress + { + Status = status, + Progress = progress + }; + } + } + } + } + } + } + } + + return result; } public QueryResult GetJobItems(SyncJobItemQuery query) @@ -795,19 +874,5 @@ namespace MediaBrowser.Server.Implementations.Sync return info; } - - private SyncedItemProgress GetSyncedItemProgress(IDataReader reader) - { - var item = new SyncedItemProgress(); - - item.ItemId = reader.GetString(0); - - if (!reader.IsDBNull(1)) - { - item.Status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true); - } - - return item; - } } }