From c2d0fd99852aae31379b89591b538d075743362f Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Mon, 12 Dec 2016 03:53:25 -0500 Subject: [PATCH 1/2] update season queries --- .../Activity/ActivityRepository.cs | 11 ++- .../Data/SqliteItemRepository.cs | 70 +++++++++---------- .../Security/AuthenticationRepository.cs | 12 ++-- .../TV/TVSeriesManager.cs | 23 +++--- MediaBrowser.Controller/Entities/TV/Series.cs | 4 +- src/Emby.Server/project.json | 19 +++++ 6 files changed, 77 insertions(+), 62 deletions(-) diff --git a/Emby.Server.Implementations/Activity/ActivityRepository.cs b/Emby.Server.Implementations/Activity/ActivityRepository.cs index fda8b949b..7ac0e680c 100644 --- a/Emby.Server.Implementations/Activity/ActivityRepository.cs +++ b/Emby.Server.Implementations/Activity/ActivityRepository.cs @@ -85,7 +85,7 @@ namespace Emby.Server.Implementations.Activity using (var connection = CreateConnection(true)) { var list = new List(); - int totalRecordCount = 0; + var result = new QueryResult(); var commandText = BaseActivitySelectText; var whereClauses = new List(); @@ -151,15 +151,12 @@ namespace Emby.Server.Implementations.Activity statement.TryBind("@DateCreated", minDate.Value.ToDateTimeParamValue()); } - totalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); + result.TotalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); } }, ReadTransactionMode); - return new QueryResult() - { - Items = list.ToArray(), - TotalRecordCount = totalRecordCount - }; + result.Items = list.ToArray(); + return result; } } } diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index 768f5b5a0..1f0f5a521 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -343,6 +343,9 @@ namespace Emby.Server.Implementations.Data // series "create index if not exists idx_TypeSeriesPresentationUniqueKey1 on TypedBaseItems(Type,SeriesPresentationUniqueKey,PresentationUniqueKey,SortName)", + // series next up + "create index if not exists idx_SeriesPresentationUniqueKey on TypedBaseItems(SeriesPresentationUniqueKey)", + // live tv programs "create index if not exists idx_TypeTopParentIdStartDate on TypedBaseItems(Type,TopParentId,StartDate)", @@ -2263,6 +2266,10 @@ namespace Emby.Server.Implementations.Data { return true; } + if (sortingFields.Contains(ItemSortBy.SeriesDatePlayed, StringComparer.OrdinalIgnoreCase)) + { + return true; + } if (query.IsFavoriteOrLiked.HasValue) { @@ -2656,7 +2663,7 @@ namespace Emby.Server.Implementations.Data } } - var totalRecordCount = 0; + var result = new QueryResult(); var isReturningZeroItems = query.Limit.HasValue && query.Limit <= 0; var statementTexts = new List(); @@ -2714,23 +2721,23 @@ namespace Emby.Server.Implementations.Data } } } + } - if (query.EnableTotalRecordCount) + if (query.EnableTotalRecordCount) + { + using (var statement = statements[statements.Count - 1]) { - using (var statement = statements[statements.Count - 1]) + if (EnableJoinUserData(query)) { - if (EnableJoinUserData(query)) - { - statement.TryBind("@UserId", query.User.Id); - } - - BindSimilarParams(query, statement); - - // Running this again will bind the params - GetWhereClauses(query, statement); - - totalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); + statement.TryBind("@UserId", query.User.Id); } + + BindSimilarParams(query, statement); + + // Running this again will bind the params + GetWhereClauses(query, statement); + + result.TotalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); } } @@ -2738,11 +2745,8 @@ namespace Emby.Server.Implementations.Data LogQueryTime("GetItems", commandText, now); - return new QueryResult() - { - Items = list.ToArray(), - TotalRecordCount = totalRecordCount - }; + result.Items = list.ToArray(); + return result; } } } @@ -2855,7 +2859,7 @@ namespace Emby.Server.Implementations.Data } if (string.Equals(name, ItemSortBy.SeriesDatePlayed, StringComparison.OrdinalIgnoreCase)) { - return new Tuple("(Select MAX(LastPlayedDate) from TypedBaseItems B" + GetJoinUserDataText(query) + " where B.Guid in (Select ItemId from AncestorIds where AncestorId in (select guid from typedbaseitems c where C.Type = 'MediaBrowser.Controller.Entities.TV.Series' And C.Guid in (Select AncestorId from AncestorIds where ItemId=A.Guid))))", false); + return new Tuple("(Select MAX(LastPlayedDate) from TypedBaseItems B" + GetJoinUserDataText(query) + " where B.SeriesPresentationUniqueKey=A.PresentationUniqueKey)", false); } return new Tuple(name, false); @@ -3094,7 +3098,7 @@ namespace Emby.Server.Implementations.Data { using (var connection = CreateConnection(true)) { - var totalRecordCount = 0; + var result = new QueryResult(); connection.RunInTransaction(db => { @@ -3136,7 +3140,7 @@ namespace Emby.Server.Implementations.Data // Running this again will bind the params GetWhereClauses(query, statement); - totalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); + result.TotalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); } } @@ -3144,11 +3148,8 @@ namespace Emby.Server.Implementations.Data LogQueryTime("GetItemIds", commandText, now); - return new QueryResult() - { - Items = list.ToArray(), - TotalRecordCount = totalRecordCount - }; + result.Items = list.ToArray(); + return result; } } } @@ -5026,7 +5027,7 @@ namespace Emby.Server.Implementations.Data var isReturningZeroItems = query.Limit.HasValue && query.Limit <= 0; var list = new List>(); - var count = 0; + var result = new QueryResult>(); var statementTexts = new List(); if (!isReturningZeroItems) @@ -5106,7 +5107,7 @@ namespace Emby.Server.Implementations.Data GetWhereClauses(innerQuery, statement); GetWhereClauses(outerQuery, statement); - count = statement.ExecuteQuery().SelectScalarInt().First(); + result.TotalRecordCount = statement.ExecuteQuery().SelectScalarInt().First(); LogQueryTime("GetItemValues", commandText, now); } @@ -5115,16 +5116,13 @@ namespace Emby.Server.Implementations.Data } } - if (count == 0) + if (result.TotalRecordCount == 0) { - count = list.Count; + result.TotalRecordCount = list.Count; } + result.Items = list.ToArray(); - return new QueryResult> - { - Items = list.ToArray(), - TotalRecordCount = count - }; + return result; } private ItemCounts GetItemCounts(IReadOnlyList reader, int countStartColumn, List typesToCount) diff --git a/Emby.Server.Implementations/Security/AuthenticationRepository.cs b/Emby.Server.Implementations/Security/AuthenticationRepository.cs index a136701da..392db6935 100644 --- a/Emby.Server.Implementations/Security/AuthenticationRepository.cs +++ b/Emby.Server.Implementations/Security/AuthenticationRepository.cs @@ -201,12 +201,13 @@ namespace Emby.Server.Implementations.Security } var list = new List(); - int totalRecordCount = 0; using (WriteLock.Read()) { using (var connection = CreateConnection(true)) { + var result = new QueryResult(); + connection.RunInTransaction(db => { var statementTexts = new List(); @@ -229,7 +230,7 @@ namespace Emby.Server.Implementations.Security { BindAuthenticationQueryParams(query, totalCountStatement); - totalRecordCount = totalCountStatement.ExecuteQuery() + result.TotalRecordCount = totalCountStatement.ExecuteQuery() .SelectScalarInt() .First(); } @@ -237,11 +238,8 @@ namespace Emby.Server.Implementations.Security }, ReadTransactionMode); - return new QueryResult() - { - Items = list.ToArray(), - TotalRecordCount = totalRecordCount - }; + result.Items = list.ToArray(); + return result; } } } diff --git a/Emby.Server.Implementations/TV/TVSeriesManager.cs b/Emby.Server.Implementations/TV/TVSeriesManager.cs index 88d224525..363fe5593 100644 --- a/Emby.Server.Implementations/TV/TVSeriesManager.cs +++ b/Emby.Server.Implementations/TV/TVSeriesManager.cs @@ -58,7 +58,8 @@ namespace Emby.Server.Implementations.TV var items = _libraryManager.GetItemList(new InternalItemsQuery(user) { IncludeItemTypes = new[] { typeof(Series).Name }, - SortOrder = SortOrder.Ascending, + SortBy = new[] { ItemSortBy.SeriesDatePlayed }, + SortOrder = SortOrder.Descending, PresentationUniqueKey = presentationUniqueKey, Limit = limit, ParentId = parentIdGuid, @@ -71,7 +72,7 @@ namespace Emby.Server.Implementations.TV } } - }).Cast(); + }).Cast().Select(GetUniqueSeriesKey); // Avoid implicitly captured closure var episodes = GetNextUpEpisodes(request, user, items); @@ -109,19 +110,20 @@ namespace Emby.Server.Implementations.TV var items = _libraryManager.GetItemList(new InternalItemsQuery(user) { IncludeItemTypes = new[] { typeof(Series).Name }, - SortOrder = SortOrder.Ascending, + SortBy = new[] { ItemSortBy.SeriesDatePlayed }, + SortOrder = SortOrder.Descending, PresentationUniqueKey = presentationUniqueKey, Limit = limit, DtoOptions = new MediaBrowser.Controller.Dto.DtoOptions { Fields = new List { - + }, EnableImages = false } - }, parentsFolders.Cast().ToList()).Cast(); + }, parentsFolders.Cast().ToList()).Cast().Select(GetUniqueSeriesKey); // Avoid implicitly captured closure var episodes = GetNextUpEpisodes(request, user, items); @@ -129,15 +131,15 @@ namespace Emby.Server.Implementations.TV return GetResult(episodes, null, request); } - public IEnumerable GetNextUpEpisodes(NextUpQuery request, User user, IEnumerable series) + public IEnumerable GetNextUpEpisodes(NextUpQuery request, User user, IEnumerable seriesKeys) { // Avoid implicitly captured closure var currentUser = user; - var allNextUp = series - .Select(i => GetNextUp(GetUniqueSeriesKey(i), currentUser)) - // Include if an episode was found, and either the series is not unwatched or the specific series was requested - .OrderByDescending(i => i.Item1); + var allNextUp = seriesKeys + .Select(i => GetNextUp(i, currentUser)); + + //allNextUp = allNextUp.OrderByDescending(i => i.Item1); // If viewing all next up for all series, remove first episodes // But if that returns empty, keep those first episodes (avoid completely empty view) @@ -160,7 +162,6 @@ namespace Emby.Server.Implementations.TV return true; }) - .Take(request.Limit.HasValue ? (request.Limit.Value * 2) : int.MaxValue) .Select(i => i.Item2()) .Where(i => i != null) .Take(request.Limit ?? int.MaxValue); diff --git a/MediaBrowser.Controller/Entities/TV/Series.cs b/MediaBrowser.Controller/Entities/TV/Series.cs index 92cd20769..caebff16f 100644 --- a/MediaBrowser.Controller/Entities/TV/Series.cs +++ b/MediaBrowser.Controller/Entities/TV/Series.cs @@ -171,7 +171,9 @@ namespace MediaBrowser.Controller.Entities.TV } query.IsVirtualItem = false; query.Limit = 0; - return LibraryManager.GetItemsResult(query).TotalRecordCount; + var totalRecordCount = LibraryManager.GetItemsResult(query).TotalRecordCount; + + return totalRecordCount; } /// diff --git a/src/Emby.Server/project.json b/src/Emby.Server/project.json index 35ebdc213..5b64beae6 100644 --- a/src/Emby.Server/project.json +++ b/src/Emby.Server/project.json @@ -18,6 +18,25 @@ "SQLitePCLRaw.provider.sqlite3.netstandard11": "1.1.1-pre20161109081005" }, + "runtimes": { + "win7-x86": {}, + "win7-x64": {}, + "win8-x86": {}, + "win8-x64": {}, + "win8-arm": {}, + "win81-x86": {}, + "win81-x64": {}, + "win81-arm": {}, + "win10-x86": {}, + "win10-x64": {}, + "win10-arm": {}, + "win10-arm64": {}, + "osx.10.10-x64": {}, + "osx.10.11-x64": {}, + "osx.10.12-x64": {}, + "ubuntu.14.04-x64": {} + }, + "frameworks": { "netcoreapp1.1": { "imports": "dnxcore50", From d84bb7160f8c3cfc953cdece890264d70e504e39 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Mon, 12 Dec 2016 14:40:27 -0500 Subject: [PATCH 2/2] update next up queries --- .../Data/SqliteItemRepository.cs | 62 ++++++++++++++++++- .../Library/LibraryManager.cs | 20 ++++++ MediaBrowser.Controller/Entities/TV/Series.cs | 28 ++++++--- .../Library/ILibraryManager.cs | 1 + .../Persistence/IItemRepository.cs | 2 + 5 files changed, 104 insertions(+), 9 deletions(-) diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index 1f0f5a521..14b81022d 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -2443,6 +2443,66 @@ namespace Emby.Server.Implementations.Data return " from TypedBaseItems " + alias; } + public int GetCount(InternalItemsQuery query) + { + if (query == null) + { + throw new ArgumentNullException("query"); + } + + CheckDisposed(); + + //Logger.Info("GetItemList: " + _environmentInfo.StackTrace); + + var now = DateTime.UtcNow; + + // Hack for right now since we currently don't support filtering out these duplicates within a query + if (query.Limit.HasValue && query.EnableGroupByMetadataKey) + { + query.Limit = query.Limit.Value + 4; + } + + var commandText = "select " + string.Join(",", GetFinalColumnsToSelect(query, new [] { "count(distinct PresentationUniqueKey)" })) + GetFromText(); + commandText += GetJoinUserDataText(query); + + var whereClauses = GetWhereClauses(query, null); + + var whereText = whereClauses.Count == 0 ? + string.Empty : + " where " + string.Join(" AND ", whereClauses.ToArray()); + + commandText += whereText; + + //commandText += GetGroupBy(query); + + int count = 0; + + using (WriteLock.Read()) + { + using (var connection = CreateConnection(true)) + { + using (var statement = PrepareStatementSafe(connection, commandText)) + { + if (EnableJoinUserData(query)) + { + statement.TryBind("@UserId", query.User.Id); + } + + BindSimilarParams(query, statement); + + // Running this again will bind the params + GetWhereClauses(query, statement); + + count = statement.ExecuteQuery().SelectScalarInt().First(); + } + } + + LogQueryTime("GetCount", commandText, now); + } + + return count; + } + public List GetItemList(InternalItemsQuery query) { if (query == null) @@ -2859,7 +2919,7 @@ namespace Emby.Server.Implementations.Data } if (string.Equals(name, ItemSortBy.SeriesDatePlayed, StringComparison.OrdinalIgnoreCase)) { - return new Tuple("(Select MAX(LastPlayedDate) from TypedBaseItems B" + GetJoinUserDataText(query) + " where B.SeriesPresentationUniqueKey=A.PresentationUniqueKey)", false); + return new Tuple("(Select MAX(LastPlayedDate) from TypedBaseItems B" + GetJoinUserDataText(query) + " where Played=1 and B.SeriesPresentationUniqueKey=A.PresentationUniqueKey)", false); } return new Tuple(name, false); diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 3db8c7213..ad91988e5 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1279,6 +1279,26 @@ namespace Emby.Server.Implementations.Library return ItemRepository.GetItemList(query); } + public int GetCount(InternalItemsQuery query) + { + if (query.Recursive && query.ParentId.HasValue) + { + var parent = GetItemById(query.ParentId.Value); + if (parent != null) + { + SetTopParentIdsOrAncestors(query, new List { parent }); + query.ParentId = null; + } + } + + if (query.User != null) + { + AddUserToQuery(query, query.User); + } + + return ItemRepository.GetCount(query); + } + public IEnumerable GetItemList(InternalItemsQuery query, List parents) { SetTopParentIdsOrAncestors(query, parents); diff --git a/MediaBrowser.Controller/Entities/TV/Series.cs b/MediaBrowser.Controller/Entities/TV/Series.cs index caebff16f..872011ce8 100644 --- a/MediaBrowser.Controller/Entities/TV/Series.cs +++ b/MediaBrowser.Controller/Entities/TV/Series.cs @@ -138,16 +138,24 @@ namespace MediaBrowser.Controller.Entities.TV var enableSeriesPresentationKey = ConfigurationManager.Configuration.EnableSeriesPresentationUniqueKey; var seriesKey = GetUniqueSeriesKey(this); - var result = LibraryManager.GetItemsResult(new InternalItemsQuery(user) + var result = LibraryManager.GetCount(new InternalItemsQuery(user) { AncestorWithPresentationUniqueKey = enableSeriesPresentationKey ? null : seriesKey, SeriesPresentationUniqueKey = enableSeriesPresentationKey ? seriesKey : null, IncludeItemTypes = new[] { typeof(Season).Name }, IsVirtualItem = false, - Limit = 0 + Limit = 0, + DtoOptions = new Dto.DtoOptions + { + Fields = new List + { + + }, + EnableImages = false + } }); - return result.TotalRecordCount; + return result; } public override int GetRecursiveChildCount(User user) @@ -159,19 +167,23 @@ namespace MediaBrowser.Controller.Entities.TV { AncestorWithPresentationUniqueKey = enableSeriesPresentationKey ? null : seriesKey, SeriesPresentationUniqueKey = enableSeriesPresentationKey ? seriesKey : null, + DtoOptions = new Dto.DtoOptions + { + Fields = new List + { + + }, + EnableImages = false + } }; - if (query.SortBy.Length == 0) - { - query.SortBy = new[] { ItemSortBy.SortName }; - } if (query.IncludeItemTypes.Length == 0) { query.IncludeItemTypes = new[] { typeof(Episode).Name, typeof(Season).Name }; } query.IsVirtualItem = false; query.Limit = 0; - var totalRecordCount = LibraryManager.GetItemsResult(query).TotalRecordCount; + var totalRecordCount = LibraryManager.GetCount(query); return totalRecordCount; } diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 955230b8a..d297fd006 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -571,5 +571,6 @@ namespace MediaBrowser.Controller.Library void RegisterIgnoredPath(string path); void UnRegisterIgnoredPath(string path); + int GetCount(InternalItemsQuery query); } } \ No newline at end of file diff --git a/MediaBrowser.Controller/Persistence/IItemRepository.cs b/MediaBrowser.Controller/Persistence/IItemRepository.cs index 7a822e7b4..58ae1f3b0 100644 --- a/MediaBrowser.Controller/Persistence/IItemRepository.cs +++ b/MediaBrowser.Controller/Persistence/IItemRepository.cs @@ -163,6 +163,8 @@ namespace MediaBrowser.Controller.Persistence /// Task. Task UpdateInheritedValues(CancellationToken cancellationToken); + int GetCount(InternalItemsQuery query); + QueryResult> GetGenres(InternalItemsQuery query); QueryResult> GetMusicGenres(InternalItemsQuery query); QueryResult> GetGameGenres(InternalItemsQuery query);