lgladsden/jellyfin-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reduce sync db access

Browse Source
This commit is contained in:
Luke Pulverenti 2016-08-03 02:38:19 -04:00
parent 7e1a0baf9b
commit 04d480297d
9 changed files with 145 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
MediaBrowser.Api/UserLibrary/BaseItemsByNameService.cs
View File

@ -180,9 +180,10 @@ namespace MediaBrowser.Api.UserLibrary
var result = GetItems(request, query); var result = GetItems(request, query);
var syncProgess = DtoService.GetSyncedItemProgress(dtoOptions);
var dtos = result.Items.Select(i => var dtos = result.Items.Select(i =>
{ {
var dto = DtoService.GetItemByNameDto(i.Item1, dtoOptions, null, user); var dto = DtoService.GetItemByNameDto(i.Item1, dtoOptions, null, syncProgess, user);
if (!string.IsNullOrWhiteSpace(request.IncludeItemTypes)) if (!string.IsNullOrWhiteSpace(request.IncludeItemTypes))
{ {
@ -325,7 +326,8 @@ namespace MediaBrowser.Api.UserLibrary
tuples = ibnItems.Select(i => new Tuple<BaseItem, List<BaseItem>>(i, new List<BaseItem>())); tuples = ibnItems.Select(i => new Tuple<BaseItem, List<BaseItem>>(i, new List<BaseItem>()));
} }
var dtos = tuples.Select(i => DtoService.GetItemByNameDto(i.Item1, dtoOptions, i.Item2, user)); var syncProgess = DtoService.GetSyncedItemProgress(dtoOptions);
var dtos = tuples.Select(i => DtoService.GetItemByNameDto(i.Item1, dtoOptions, i.Item2, syncProgess, user));
result.Items = dtos.Where(i => i != null).ToArray(); result.Items = dtos.Where(i => i != null).ToArray();

11
MediaBrowser.Controller/Dto/IDtoService.cs
View File

@ -4,6 +4,8 @@ using MediaBrowser.Model.Dto;
using MediaBrowser.Model.Querying; using MediaBrowser.Model.Querying;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using MediaBrowser.Controller.Sync;
using MediaBrowser.Model.Sync;
namespace MediaBrowser.Controller.Dto namespace MediaBrowser.Controller.Dto
{ {
@ -89,11 +91,8 @@ namespace MediaBrowser.Controller.Dto
/// <summary> /// <summary>
/// Gets the item by name dto. /// Gets the item by name dto.
/// </summary> /// </summary>
/// <param name="item">The item.</param> BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List<BaseItem> taggedItems, Dictionary<string, SyncedItemProgress> syncProgress, User user = null);
/// <param name="options">The options.</param>
/// <param name="taggedItems">The tagged items.</param> Dictionary<string, SyncedItemProgress> GetSyncedItemProgress(DtoOptions options);
/// <param name="user">The user.</param>
/// <returns>BaseItemDto.</returns>
BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List<BaseItem> taggedItems, User user = null);
} }
} }

2
MediaBrowser.Controller/Sync/ISyncManager.cs
View File

@ -151,7 +151,7 @@ namespace MediaBrowser.Controller.Sync
/// </summary> /// </summary>
/// <param name="query">The query.</param> /// <param name="query">The query.</param>
/// <returns>QueryResult&lt;System.String&gt;.</returns> /// <returns>QueryResult&lt;System.String&gt;.</returns>
QueryResult<SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query); Dictionary<string, SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query);
/// <summary> /// <summary>
/// Reports the synchronize job item transfer beginning. /// Reports the synchronize job item transfer beginning.

5
MediaBrowser.Controller/Sync/ISyncRepository.cs
View File

@ -1,4 +1,5 @@
using MediaBrowser.Model.Querying; using System.Collections.Generic;
using MediaBrowser.Model.Querying;
using MediaBrowser.Model.Sync; using MediaBrowser.Model.Sync;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -74,6 +75,6 @@ namespace MediaBrowser.Controller.Sync
/// </summary> /// </summary>
/// <param name="query">The query.</param> /// <param name="query">The query.</param>
/// <returns>QueryResult&lt;System.String&gt;.</returns> /// <returns>QueryResult&lt;System.String&gt;.</returns>
QueryResult<SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query); Dictionary<string, SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query);
} }
} }

2
MediaBrowser.Controller/Sync/SyncedItemProgress.cs
View File

@ -4,7 +4,7 @@ namespace MediaBrowser.Controller.Sync
{ {
public class SyncedItemProgress public class SyncedItemProgress
{ {
public string ItemId { get; set; } public double Progress { get; set; }
public SyncJobItemStatus Status { get; set; } public SyncJobItemStatus Status { get; set; }
} }
} }

1
MediaBrowser.Model/Dto/IHasSyncInfo.cs
View File

@ -7,6 +7,7 @@ namespace MediaBrowser.Model.Dto
string Id { get; } string Id { get; }
bool? SupportsSync { get; set; } bool? SupportsSync { get; set; }
bool? HasSyncJob { get; set; } bool? HasSyncJob { get; set; }
double? SyncPercent { get; set; }
bool? IsSynced { get; set; } bool? IsSynced { get; set; }
SyncJobItemStatus? SyncStatus { get; set; } SyncJobItemStatus? SyncStatus { get; set; }
} }

111
MediaBrowser.Server.Implementations/Dto/DtoService.cs
View File

@ -97,8 +97,7 @@ namespace MediaBrowser.Server.Implementations.Dto
throw new ArgumentNullException("options"); throw new ArgumentNullException("options");
} }
var syncJobItems = GetSyncedItemProgress(options); var syncDictionary = GetSyncedItemProgress(options);
var syncDictionary = GetSyncedItemProgressDictionary(syncJobItems);
var list = new List<BaseItemDto>(); var list = new List<BaseItemDto>();
var programTuples = new List<Tuple<BaseItem, BaseItemDto>>(); var programTuples = new List<Tuple<BaseItem, BaseItemDto>>();
@ -106,7 +105,7 @@ namespace MediaBrowser.Server.Implementations.Dto
foreach (var item in items) foreach (var item in items)
{ {
var dto = await GetBaseItemDtoInternal(item, options, syncDictionary, user, owner).ConfigureAwait(false); var dto = await GetBaseItemDtoInternal(item, options, user, owner).ConfigureAwait(false);
var tvChannel = item as LiveTvChannel; var tvChannel = item as LiveTvChannel;
if (tvChannel != null) if (tvChannel != null)
@ -133,7 +132,7 @@ namespace MediaBrowser.Server.Implementations.Dto
} }
} }
FillSyncInfo(dto, item, options, user, syncJobItems); FillSyncInfo(dto, item, options, user, syncDictionary);
list.Add(dto); list.Add(dto);
} }
@ -151,23 +150,11 @@ namespace MediaBrowser.Server.Implementations.Dto
return list; return list;
} }
private Dictionary<string, SyncedItemProgress> GetSyncedItemProgressDictionary(IEnumerable<SyncedItemProgress> items)
{
var dict = new Dictionary<string, SyncedItemProgress>();
foreach (var item in items)
{
dict[item.ItemId] = item;
}
return dict;
}
public BaseItemDto GetBaseItemDto(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null) public BaseItemDto GetBaseItemDto(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null)
{ {
var syncProgress = GetSyncedItemProgress(options); var syncDictionary = GetSyncedItemProgress(options);
var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user, owner).Result; var dto = GetBaseItemDtoInternal(item, options, user, owner).Result;
var tvChannel = item as LiveTvChannel; var tvChannel = item as LiveTvChannel;
if (tvChannel != null) if (tvChannel != null)
{ {
@ -190,11 +177,11 @@ namespace MediaBrowser.Server.Implementations.Dto
SetItemByNameInfo(item, dto, GetTaggedItems(byName, user), user); SetItemByNameInfo(item, dto, GetTaggedItems(byName, user), user);
} }
FillSyncInfo(dto, item, options, user, syncProgress); FillSyncInfo(dto, item, options, user, syncDictionary);
return dto; return dto;
} }
FillSyncInfo(dto, item, options, user, syncProgress); FillSyncInfo(dto, item, options, user, syncDictionary);
return dto; return dto;
} }
@ -210,23 +197,24 @@ namespace MediaBrowser.Server.Implementations.Dto
return items; return items;
} }
private SyncedItemProgress[] GetSyncedItemProgress(DtoOptions options) public Dictionary<string, SyncedItemProgress> GetSyncedItemProgress(DtoOptions options)
{ {
if (!options.Fields.Contains(ItemFields.SyncInfo)) if (!options.Fields.Contains(ItemFields.BasicSyncInfo) &&
!options.Fields.Contains(ItemFields.SyncInfo))
{ {
return new SyncedItemProgress[] { }; return new Dictionary<string, SyncedItemProgress>();
} }
var deviceId = options.DeviceId; var deviceId = options.DeviceId;
if (string.IsNullOrWhiteSpace(deviceId)) if (string.IsNullOrWhiteSpace(deviceId))
{ {
return new SyncedItemProgress[] { }; return new Dictionary<string, SyncedItemProgress>();
} }
var caps = _deviceManager().GetCapabilities(deviceId); var caps = _deviceManager().GetCapabilities(deviceId);
if (caps == null || !caps.SupportsSync) if (caps == null || !caps.SupportsSync)
{ {
return new SyncedItemProgress[] { }; return new Dictionary<string, SyncedItemProgress>();
} }
return _syncManager.GetSyncedItemProgresses(new SyncJobItemQuery return _syncManager.GetSyncedItemProgresses(new SyncJobItemQuery
@ -240,12 +228,13 @@ namespace MediaBrowser.Server.Implementations.Dto
SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.ReadyToTransfer,
SyncJobItemStatus.Synced SyncJobItemStatus.Synced
} }
}).Items; });
} }
public void FillSyncInfo(IEnumerable<Tuple<BaseItem, BaseItemDto>> tuples, DtoOptions options, User user) public void FillSyncInfo(IEnumerable<Tuple<BaseItem, BaseItemDto>> tuples, DtoOptions options, User user)
{ {
if (options.Fields.Contains(ItemFields.SyncInfo)) if (options.Fields.Contains(ItemFields.BasicSyncInfo) ||
options.Fields.Contains(ItemFields.SyncInfo))
{ {
var syncProgress = GetSyncedItemProgress(options); var syncProgress = GetSyncedItemProgress(options);
@ -258,37 +247,40 @@ namespace MediaBrowser.Server.Implementations.Dto
} }
} }
private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, SyncedItemProgress[] syncProgress) private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, Dictionary<string, SyncedItemProgress> syncProgress)
{ {
var hasFullSyncInfo = options.Fields.Contains(ItemFields.SyncInfo); var hasFullSyncInfo = options.Fields.Contains(ItemFields.SyncInfo);
if (hasFullSyncInfo || options.Fields.Contains(ItemFields.BasicSyncInfo)) if (!options.Fields.Contains(ItemFields.BasicSyncInfo) &&
!hasFullSyncInfo)
{ {
var userCanSync = user != null && user.Policy.EnableSync; return;
dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item);
} }
if (dto.SupportsSync ?? false) if (dto.SupportsSync ?? false)
{ {
dto.HasSyncJob = syncProgress.Any(i => i.Status != SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); SyncedItemProgress syncStatus;
dto.IsSynced = syncProgress.Any(i => i.Status == SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); if (syncProgress.TryGetValue(dto.Id, out syncStatus))
if (hasFullSyncInfo)
{ {
if (dto.IsSynced.Value) if (syncStatus.Status == SyncJobItemStatus.Synced)
{ {
dto.SyncStatus = SyncJobItemStatus.Synced; dto.SyncPercent = 100;
}
else
{
dto.SyncPercent = syncStatus.Progress;
} }
else if (dto.HasSyncJob.Value) if (hasFullSyncInfo)
{ {
dto.SyncStatus = syncProgress.Where(i => string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)).Select(i => i.Status).Max(); dto.HasSyncJob = true;
dto.SyncStatus = syncStatus.Status;
} }
} }
} }
} }
private async Task<BaseItemDto> GetBaseItemDtoInternal(BaseItem item, DtoOptions options, Dictionary<string, SyncedItemProgress> syncProgress, User user = null, BaseItem owner = null) private async Task<BaseItemDto> GetBaseItemDtoInternal(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null)
{ {
var fields = options.Fields; var fields = options.Fields;
@ -337,7 +329,7 @@ namespace MediaBrowser.Server.Implementations.Dto
if (user != null) if (user != null)
{ {
await AttachUserSpecificInfo(dto, item, user, fields, syncProgress).ConfigureAwait(false); await AttachUserSpecificInfo(dto, item, user, fields).ConfigureAwait(false);
} }
var hasMediaSources = item as IHasMediaSources; var hasMediaSources = item as IHasMediaSources;
@ -400,11 +392,9 @@ namespace MediaBrowser.Server.Implementations.Dto
return dto; return dto;
} }
public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List<BaseItem> taggedItems, User user = null) public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List<BaseItem> taggedItems, Dictionary<string, SyncedItemProgress> syncProgress, User user = null)
{ {
var syncProgress = GetSyncedItemProgress(options); var dto = GetBaseItemDtoInternal(item, options, user).Result;
var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user).Result;
if (taggedItems != null && options.Fields.Contains(ItemFields.ItemCounts)) if (taggedItems != null && options.Fields.Contains(ItemFields.ItemCounts))
{ {
@ -452,28 +442,13 @@ namespace MediaBrowser.Server.Implementations.Dto
/// <param name="item">The item.</param> /// <param name="item">The item.</param>
/// <param name="user">The user.</param> /// <param name="user">The user.</param>
/// <param name="fields">The fields.</param> /// <param name="fields">The fields.</param>
/// <param name="syncProgress">The synchronize progress.</param> private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List<ItemFields> fields)
private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List<ItemFields> fields, Dictionary<string, SyncedItemProgress> syncProgress)
{ {
if (item.IsFolder) if (item.IsFolder)
{ {
var folder = (Folder)item; var folder = (Folder)item;
if (item.SourceType == SourceType.Library && folder.SupportsUserDataFromChildren && fields.Contains(ItemFields.SyncInfo)) dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false);
{
// Skip the user data manager because we've already looped through the recursive tree and don't want to do it twice
// TODO: Improve in future
dto.UserData = GetUserItemDataDto(_userDataRepository.GetUserData(user, item));
await SetSpecialCounts(folder, user, dto, fields, syncProgress).ConfigureAwait(false);
dto.UserData.Played = dto.UserData.PlayedPercentage.HasValue &&
dto.UserData.PlayedPercentage.Value >= 100;
}
else
{
dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false);
}
if (item.SourceType == SourceType.Library) if (item.SourceType == SourceType.Library)
{ {
@ -498,6 +473,12 @@ namespace MediaBrowser.Server.Implementations.Dto
dto.PlayAccess = item.GetPlayAccess(user); dto.PlayAccess = item.GetPlayAccess(user);
if (fields.Contains(ItemFields.BasicSyncInfo) || fields.Contains(ItemFields.SyncInfo))
{
var userCanSync = user != null && user.Policy.EnableSync;
dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item);
}
if (fields.Contains(ItemFields.SeasonUserData)) if (fields.Contains(ItemFields.SeasonUserData))
{ {
var episode = item as Episode; var episode = item as Episode;
@ -1531,7 +1512,7 @@ namespace MediaBrowser.Server.Implementations.Dto
/// <param name="fields">The fields.</param> /// <param name="fields">The fields.</param>
/// <param name="syncProgress">The synchronize progress.</param> /// <param name="syncProgress">The synchronize progress.</param>
/// <returns>Task.</returns> /// <returns>Task.</returns>
private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List<ItemFields> fields, Dictionary<string, SyncedItemProgress> syncProgress) private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List<ItemFields> fields, Dictionary<string, SyncJobItemStatus> syncProgress)
{ {
var recursiveItemCount = 0; var recursiveItemCount = 0;
var unplayed = 0; var unplayed = 0;
@ -1580,10 +1561,10 @@ namespace MediaBrowser.Server.Implementations.Dto
} }
double percent = 0; double percent = 0;
SyncedItemProgress syncItemProgress; SyncJobItemStatus syncItemProgress;
if (syncProgress.TryGetValue(child.Id.ToString("N"), out syncItemProgress)) if (syncProgress.TryGetValue(child.Id.ToString("N"), out syncItemProgress))
{ {
switch (syncItemProgress.Status) switch (syncItemProgress)
{ {
case SyncJobItemStatus.Synced: case SyncJobItemStatus.Synced:
percent = 100; percent = 100;

2
MediaBrowser.Server.Implementations/Sync/SyncManager.cs
View File

@ -1124,7 +1124,7 @@ namespace MediaBrowser.Server.Implementations.Sync
await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false);
} }
public QueryResult<SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query) public Dictionary<string, SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query)
{ {
return _repo.GetSyncedItemProgresses(query); return _repo.GetSyncedItemProgresses(query);
} }

99
MediaBrowser.Server.Implementations/Sync/SyncRepository.cs
View File

@ -29,6 +29,17 @@ namespace MediaBrowser.Server.Implementations.Sync
DbFilePath = Path.Combine(appPaths.DataPath, "sync14.db"); DbFilePath = Path.Combine(appPaths.DataPath, "sync14.db");
} }
private class SyncSummary
{
public Dictionary<string, int> Items { get; set; }
public SyncSummary()
{
Items = new Dictionary<string, int>();
}
}
public async Task Initialize() public async Task Initialize()
{ {
using (var connection = await CreateConnection().ConfigureAwait(false)) using (var connection = await CreateConnection().ConfigureAwait(false))
@ -41,7 +52,10 @@ namespace MediaBrowser.Server.Implementations.Sync
"create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, TemporaryPath TEXT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT, AdditionalFiles TEXT, MediaSource TEXT, IsMarkedForRemoval BIT, JobItemIndex INT, ItemDateModifiedTicks BIGINT)", "create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, TemporaryPath TEXT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT, AdditionalFiles TEXT, MediaSource TEXT, IsMarkedForRemoval BIT, JobItemIndex INT, ItemDateModifiedTicks BIGINT)",
"create index if not exists idx_SyncJobItems1 on SyncJobItems(Id)", "create index if not exists idx_SyncJobItems1 on SyncJobItems(Id)",
"create index if not exists idx_SyncJobItems2 on SyncJobItems(TargetId)",
"drop index if exists idx_SyncJobItems2",
"drop index if exists idx_SyncJobItems3",
"create index if not exists idx_SyncJobItems4 on SyncJobItems(TargetId,ItemId,Status,Progress,DateCreated)",
"pragma shrink_memory" "pragma shrink_memory"
}; };
@ -583,9 +597,74 @@ namespace MediaBrowser.Server.Implementations.Sync
} }
} }
public QueryResult<SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query) public Dictionary<string, SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query)
{ {
return GetJobItemReader(query, "select ItemId,Status from SyncJobItems", GetSyncedItemProgress); var result = new Dictionary<string, SyncedItemProgress>();
using (var connection = CreateConnection(true).Result)
{
using (var cmd = connection.CreateCommand())
{
cmd.CommandText = "select ItemId,Progress,Status from SyncJobItems";
var whereClauses = new List<string>();
if (!string.IsNullOrWhiteSpace(query.TargetId))
{
whereClauses.Add("TargetId=@TargetId");
cmd.Parameters.Add(cmd, "@TargetId", DbType.String).Value = query.TargetId;
}
if (query.Statuses.Length > 0)
{
var statuses = string.Join(",", query.Statuses.Select(i => "'" + i.ToString() + "'").ToArray());
whereClauses.Add(string.Format("Status in ({0})", statuses));
}
if (whereClauses.Count > 0)
{
cmd.CommandText += " where " + string.Join(" AND ", whereClauses.ToArray());
}
using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult))
{
while (reader.Read())
{
var itemId = reader.GetString(0);
if (!reader.IsDBNull(1))
{
var status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true);
if (status == SyncJobItemStatus.Synced)
{
result[itemId] = new SyncedItemProgress
{
Status = SyncJobItemStatus.Synced
};
}
else
{
SyncedItemProgress currentStatus;
double progress = reader.IsDBNull(2) ? 0.0 : reader.GetDouble(2);
if (!result.TryGetValue(itemId, out currentStatus) || (currentStatus.Status != SyncJobItemStatus.Synced && progress >= currentStatus.Progress))
{
result[itemId] = new SyncedItemProgress
{
Status = status,
Progress = progress
};
}
}
}
}
}
}
}
return result;
} }
public QueryResult<SyncJobItem> GetJobItems(SyncJobItemQuery query) public QueryResult<SyncJobItem> GetJobItems(SyncJobItemQuery query)
@ -795,19 +874,5 @@ namespace MediaBrowser.Server.Implementations.Sync
return info; return info;
} }
private SyncedItemProgress GetSyncedItemProgress(IDataReader reader)
{
var item = new SyncedItemProgress();
item.ItemId = reader.GetString(0);
if (!reader.IsDBNull(1))
{
item.Status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true);
}
return item;
}
} }
} }