From e01202030dcd16cd9c7c3327b4e411be7de02614 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Tue, 21 May 2013 11:52:59 -0400 Subject: [PATCH] removed sql delayed writer in favor of prepared statements --- .../Providers/TV/RemoteSeasonProvider.cs | 1 - .../Providers/TV/RemoteSeriesProvider.cs | 2 +- .../SQLiteDisplayPreferencesRepository.cs | 18 +- .../Sqlite/SQLiteItemRepository.cs | 169 +++++++++++++++--- .../Sqlite/SQLiteRepository.cs | 165 ++--------------- .../Sqlite/SQLiteUserDataRepository.cs | 18 +- .../Sqlite/SQLiteUserRepository.cs | 20 +-- 7 files changed, 166 insertions(+), 227 deletions(-) diff --git a/MediaBrowser.Controller/Providers/TV/RemoteSeasonProvider.cs b/MediaBrowser.Controller/Providers/TV/RemoteSeasonProvider.cs index e9953d135..9b0bf824a 100644 --- a/MediaBrowser.Controller/Providers/TV/RemoteSeasonProvider.cs +++ b/MediaBrowser.Controller/Providers/TV/RemoteSeasonProvider.cs @@ -279,7 +279,6 @@ namespace MediaBrowser.Controller.Providers.TV n = n.SelectSingleNode("./BannerPath"); if (n != null) { - if (season.BackdropImagePaths == null) season.BackdropImagePaths = new List(); season.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(season, TVUtils.BannerUrl + n.InnerText, "backdrop" + Path.GetExtension(n.InnerText), ConfigurationManager.Configuration.SaveLocalMeta, RemoteSeriesProvider.Current.TvDbResourcePool, cancellationToken).ConfigureAwait(false)); } } diff --git a/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs b/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs index f6a77026b..a50dda066 100644 --- a/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs +++ b/MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs @@ -525,7 +525,7 @@ namespace MediaBrowser.Controller.Providers.TV if (series.BackdropImagePaths.Count < ConfigurationManager.Configuration.MaxBackdrops) { - var bdNo = 0; + var bdNo = series.BackdropImagePaths.Count; var xmlNodeList = images.SelectNodes("//Banner[BannerType='fanart']"); if (xmlNodeList != null) { diff --git a/MediaBrowser.Server.Implementations/Sqlite/SQLiteDisplayPreferencesRepository.cs b/MediaBrowser.Server.Implementations/Sqlite/SQLiteDisplayPreferencesRepository.cs index 93bb174c6..93b246380 100644 --- a/MediaBrowser.Server.Implementations/Sqlite/SQLiteDisplayPreferencesRepository.cs +++ b/MediaBrowser.Server.Implementations/Sqlite/SQLiteDisplayPreferencesRepository.cs @@ -33,18 +33,6 @@ namespace MediaBrowser.Server.Implementations.Sqlite } } - /// - /// Gets a value indicating whether [enable delayed commands]. - /// - /// true if [enable delayed commands]; otherwise, false. - protected override bool EnableDelayedCommands - { - get - { - return false; - } - } - /// /// The _json serializer /// @@ -132,13 +120,13 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "replace into displaypreferences (id, data) values (@1, @2)"; cmd.AddParam("@1", displayPreferences.Id); cmd.AddParam("@2", serialized); - using (var tran = connection.BeginTransaction()) + using (var tran = Connection.BeginTransaction()) { try { @@ -174,7 +162,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException("displayPreferencesId"); } - var cmd = connection.CreateCommand(); + var cmd = Connection.CreateCommand(); cmd.CommandText = "select data from displaypreferences where id = @id"; var idParam = cmd.Parameters.Add("@id", DbType.Guid); diff --git a/MediaBrowser.Server.Implementations/Sqlite/SQLiteItemRepository.cs b/MediaBrowser.Server.Implementations/Sqlite/SQLiteItemRepository.cs index 62268c0c3..23ab10499 100644 --- a/MediaBrowser.Server.Implementations/Sqlite/SQLiteItemRepository.cs +++ b/MediaBrowser.Server.Implementations/Sqlite/SQLiteItemRepository.cs @@ -8,6 +8,7 @@ using MediaBrowser.Server.Implementations.Reflection; using System; using System.Collections.Generic; using System.Data; +using System.Data.SQLite; using System.IO; using System.Linq; using System.Threading; @@ -53,6 +54,19 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// private readonly IApplicationPaths _appPaths; + /// + /// The _save item command + /// + private SQLiteCommand _saveItemCommand; + /// + /// The _delete children command + /// + private SQLiteCommand _deleteChildrenCommand; + /// + /// The _save children command + /// + private SQLiteCommand _saveChildrenCommand; + /// /// Initializes a new instance of the class. /// @@ -100,6 +114,8 @@ namespace MediaBrowser.Server.Implementations.Sqlite }; RunQueries(queries); + + PrepareStatements(); } //cascade delete triggers @@ -116,6 +132,39 @@ namespace MediaBrowser.Server.Implementations.Sqlite DELETE FROM children WHERE children.child = old.child; END"; + /// + /// The _write lock + /// + private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1,1); + + /// + /// Prepares the statements. + /// + private void PrepareStatements() + { + _saveItemCommand = new SQLiteCommand + { + CommandText = "replace into items (guid, obj_type, data) values (@1, @2, @3)" + }; + + _saveItemCommand.Parameters.Add(new SQLiteParameter("@1")); + _saveItemCommand.Parameters.Add(new SQLiteParameter("@2")); + _saveItemCommand.Parameters.Add(new SQLiteParameter("@3")); + + _deleteChildrenCommand = new SQLiteCommand + { + CommandText = "delete from children where guid = @guid" + }; + _deleteChildrenCommand.Parameters.Add(new SQLiteParameter("@guid")); + + _saveChildrenCommand = new SQLiteCommand + { + CommandText = "replace into children (guid, child) values (@guid, @child)" + }; + _saveChildrenCommand.Parameters.Add(new SQLiteParameter("@guid")); + _saveChildrenCommand.Parameters.Add(new SQLiteParameter("@child")); + } + /// /// Save a standard item in the repo /// @@ -123,7 +172,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// The cancellation token. /// Task. /// item - public Task SaveItem(BaseItem item, CancellationToken cancellationToken) + public async Task SaveItem(BaseItem item, CancellationToken cancellationToken) { if (item == null) { @@ -137,19 +186,51 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - return Task.Run(() => + var serialized = _jsonSerializer.SerializeToBytes(item); + + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + SQLiteTransaction transaction = null; + + try { - var serialized = _jsonSerializer.SerializeToBytes(item); + transaction = Connection.BeginTransaction(); - cancellationToken.ThrowIfCancellationRequested(); + _saveItemCommand.Parameters[0].Value = item.Id; + _saveItemCommand.Parameters[1].Value = item.GetType().FullName; + _saveItemCommand.Parameters[2].Value = serialized; - var cmd = connection.CreateCommand(); - cmd.CommandText = "replace into items (guid, obj_type, data) values (@1, @2, @3)"; - cmd.AddParam("@1", item.Id); - cmd.AddParam("@2", item.GetType().FullName); - cmd.AddParam("@3", serialized); - QueueCommand(cmd); - }); + _saveItemCommand.Transaction = transaction; + + await _saveItemCommand.ExecuteNonQueryAsync(cancellationToken); + + transaction.Commit(); + } + catch (OperationCanceledException) + { + if (transaction != null) + { + transaction.Rollback(); + } + } + catch (Exception e) + { + Logger.ErrorException("Failed to save item:", e); + + if (transaction != null) + { + transaction.Rollback(); + } + } + finally + { + if (transaction != null) + { + transaction.Dispose(); + } + + _writeLock.Release(); + } } /// @@ -157,6 +238,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// /// The id. /// BaseItem. + /// id /// public BaseItem GetItem(Guid id) { @@ -189,6 +271,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// /// The id. /// BaseItem. + /// id /// protected BaseItem RetrieveItemInternal(Guid id) { @@ -197,7 +280,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException("id"); } - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "select obj_type,data from items where guid = @guid"; var guidParam = cmd.Parameters.Add("@guid", DbType.Guid); @@ -240,7 +323,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException(); } - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "select obj_type,data from items where guid in (select child from children where guid = @guid)"; var guidParam = cmd.Parameters.Add("@guid", DbType.Guid); @@ -281,7 +364,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// The cancellation token. /// Task. /// id - public Task SaveChildren(Guid id, IEnumerable children, CancellationToken cancellationToken) + public async Task SaveChildren(Guid id, IEnumerable children, CancellationToken cancellationToken) { if (id == Guid.Empty) { @@ -300,27 +383,57 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - return Task.Run(() => + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + SQLiteTransaction transaction = null; + + try { - var cmd = connection.CreateCommand(); + transaction = Connection.BeginTransaction(); - cmd.CommandText = "delete from children where guid = @guid"; - cmd.AddParam("@guid", id); - - QueueCommand(cmd); + // Delete exising children + _deleteChildrenCommand.Parameters[0].Value = id; + _deleteChildrenCommand.Transaction = transaction; + await _deleteChildrenCommand.ExecuteNonQueryAsync(cancellationToken); + // Save new children foreach (var child in children) { - var guid = child.Id; - cmd = connection.CreateCommand(); - cmd.AddParam("@guid", id); - cmd.CommandText = "replace into children (guid, child) values (@guid, @child)"; - var childParam = cmd.Parameters.Add("@child", DbType.Guid); + _saveChildrenCommand.Transaction = transaction; + + _saveChildrenCommand.Parameters[0].Value = id; + _saveChildrenCommand.Parameters[1].Value = child.Id; - childParam.Value = guid; - QueueCommand(cmd); + await _saveChildrenCommand.ExecuteNonQueryAsync(cancellationToken); } - }); + + transaction.Commit(); + } + catch (OperationCanceledException) + { + if (transaction != null) + { + transaction.Rollback(); + } + } + catch (Exception e) + { + Logger.ErrorException("Failed to save item:", e); + + if (transaction != null) + { + transaction.Rollback(); + } + } + finally + { + if (transaction != null) + { + transaction.Dispose(); + } + + _writeLock.Release(); + } } /// diff --git a/MediaBrowser.Server.Implementations/Sqlite/SQLiteRepository.cs b/MediaBrowser.Server.Implementations/Sqlite/SQLiteRepository.cs index 6aeb63d9b..bd60c834f 100644 --- a/MediaBrowser.Server.Implementations/Sqlite/SQLiteRepository.cs +++ b/MediaBrowser.Server.Implementations/Sqlite/SQLiteRepository.cs @@ -18,24 +18,11 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// /// The db file name /// - protected string dbFileName; + protected string DbFileName; /// /// The connection /// - protected SQLiteConnection connection; - /// - /// The delayed commands - /// - protected ConcurrentQueue delayedCommands = new ConcurrentQueue(); - /// - /// The flush interval - /// - private const int FlushInterval = 2000; - - /// - /// The flush timer - /// - private Timer FlushTimer; + protected SQLiteConnection Connection; /// /// Gets the logger. @@ -43,18 +30,6 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// The logger. protected ILogger Logger { get; private set; } - /// - /// Gets a value indicating whether [enable delayed commands]. - /// - /// true if [enable delayed commands]; otherwise, false. - protected virtual bool EnableDelayedCommands - { - get - { - return true; - } - } - /// /// Initializes a new instance of the class. /// @@ -83,7 +58,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException("dbPath"); } - dbFileName = dbPath; + DbFileName = dbPath; var connectionstr = new SQLiteConnectionStringBuilder { PageSize = 4096, @@ -93,15 +68,9 @@ namespace MediaBrowser.Server.Implementations.Sqlite JournalMode = SQLiteJournalModeEnum.Memory }; - connection = new SQLiteConnection(connectionstr.ConnectionString); + Connection = new SQLiteConnection(connectionstr.ConnectionString); - await connection.OpenAsync().ConfigureAwait(false); - - if (EnableDelayedCommands) - { - // Run once - FlushTimer = new Timer(Flush, null, TimeSpan.FromMilliseconds(FlushInterval), TimeSpan.FromMilliseconds(-1)); - } + await Connection.OpenAsync().ConfigureAwait(false); } /// @@ -117,11 +86,11 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException("queries"); } - using (var tran = connection.BeginTransaction()) + using (var tran = Connection.BeginTransaction()) { try { - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { foreach (var query in queries) { @@ -165,26 +134,15 @@ namespace MediaBrowser.Server.Implementations.Sqlite { lock (_disposeLock) { - if (connection != null) + if (Connection != null) { - if (EnableDelayedCommands) + if (Connection.IsOpen()) { - FlushOnDispose(); + Connection.Close(); } - if (connection.IsOpen()) - { - connection.Close(); - } - - connection.Dispose(); - connection = null; - } - - if (FlushTimer != null) - { - FlushTimer.Dispose(); - FlushTimer = null; + Connection.Dispose(); + Connection = null; } } } @@ -195,101 +153,6 @@ namespace MediaBrowser.Server.Implementations.Sqlite } } - /// - /// Flushes the on dispose. - /// - private void FlushOnDispose() - { - // If we're not already flushing, do it now - if (!_isFlushing) - { - Flush(null); - } - - // Don't dispose in the middle of a flush - while (_isFlushing) - { - Thread.Sleep(25); - } - } - - /// - /// Queues the command. - /// - /// The CMD. - /// cmd - protected void QueueCommand(SQLiteCommand cmd) - { - if (cmd == null) - { - throw new ArgumentNullException("cmd"); - } - - delayedCommands.Enqueue(cmd); - } - - /// - /// The is flushing - /// - private bool _isFlushing; - - /// - /// Flushes the specified sender. - /// - /// The sender. - private void Flush(object sender) - { - // Cannot call Count on a ConcurrentQueue since it's an O(n) operation - // Use IsEmpty instead - if (delayedCommands.IsEmpty) - { - FlushTimer.Change(TimeSpan.FromMilliseconds(FlushInterval), TimeSpan.FromMilliseconds(-1)); - return; - } - - if (_isFlushing) - { - return; - } - - _isFlushing = true; - var numCommands = 0; - - using (var tran = connection.BeginTransaction()) - { - try - { - while (!delayedCommands.IsEmpty) - { - SQLiteCommand command; - - delayedCommands.TryDequeue(out command); - - command.Connection = connection; - command.Transaction = tran; - - command.ExecuteNonQuery(); - - command.Dispose(); - - numCommands++; - } - - tran.Commit(); - } - catch (Exception e) - { - Logger.ErrorException("Failed to commit transaction.", e); - tran.Rollback(); - } - } - - Logger.Debug("SQL Delayed writer executed " + numCommands + " commands"); - - FlushTimer.Change(TimeSpan.FromMilliseconds(FlushInterval), TimeSpan.FromMilliseconds(-1)); - _isFlushing = false; - } - /// /// Executes the command. /// @@ -303,11 +166,11 @@ namespace MediaBrowser.Server.Implementations.Sqlite throw new ArgumentNullException("cmd"); } - using (var tran = connection.BeginTransaction()) + using (var tran = Connection.BeginTransaction()) { try { - cmd.Connection = connection; + cmd.Connection = Connection; cmd.Transaction = tran; await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); diff --git a/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserDataRepository.cs b/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserDataRepository.cs index 443b15c10..d20b59035 100644 --- a/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserDataRepository.cs +++ b/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserDataRepository.cs @@ -36,18 +36,6 @@ namespace MediaBrowser.Server.Implementations.Sqlite } } - /// - /// Gets a value indicating whether [enable delayed commands]. - /// - /// true if [enable delayed commands]; otherwise, false. - protected override bool EnableDelayedCommands - { - get - { - return false; - } - } - private readonly IJsonSerializer _jsonSerializer; /// @@ -184,14 +172,14 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "replace into userdata (key, userId, data) values (@1, @2, @3)"; cmd.AddParam("@1", key); cmd.AddParam("@2", userId); cmd.AddParam("@3", serialized); - using (var tran = connection.BeginTransaction()) + using (var tran = Connection.BeginTransaction()) { try { @@ -247,7 +235,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// Task{UserItemData}. private async Task RetrieveUserData(Guid userId, string key) { - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "select data from userdata where key = @key and userId=@userId"; diff --git a/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserRepository.cs b/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserRepository.cs index eb5868932..baaf9d956 100644 --- a/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserRepository.cs +++ b/MediaBrowser.Server.Implementations/Sqlite/SQLiteUserRepository.cs @@ -45,18 +45,6 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// private readonly IApplicationPaths _appPaths; - /// - /// Gets a value indicating whether [enable delayed commands]. - /// - /// true if [enable delayed commands]; otherwise, false. - protected override bool EnableDelayedCommands - { - get - { - return false; - } - } - /// /// Initializes a new instance of the class. /// @@ -127,13 +115,13 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "replace into users (guid, data) values (@1, @2)"; cmd.AddParam("@1", user.Id); cmd.AddParam("@2", serialized); - using (var tran = connection.BeginTransaction()) + using (var tran = Connection.BeginTransaction()) { try { @@ -162,7 +150,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite /// IEnumerable{User}. public IEnumerable RetrieveAllUsers() { - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "select data from users"; @@ -201,7 +189,7 @@ namespace MediaBrowser.Server.Implementations.Sqlite cancellationToken.ThrowIfCancellationRequested(); - using (var cmd = connection.CreateCommand()) + using (var cmd = Connection.CreateCommand()) { cmd.CommandText = "delete from users where guid=@guid"; var guidParam = cmd.Parameters.Add("@guid", DbType.Guid);