Merge pull request #7604 from Jellifi007/fixes-diactritics
Co-authored-by: Cody Robibero <cody@robibe.ro>
(cherry picked from commit 8d1d973438
)
Signed-off-by: crobibero <cody@robibe.ro>
This commit is contained in:
parent
e263e9c2b1
commit
2b1a915ead
|
@ -6,8 +6,8 @@ using System.IO;
|
|||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using Diacritics.Extensions;
|
||||
using Emby.Dlna.Didl;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
|
|
|
@ -11,7 +11,6 @@ using System.Linq;
|
|||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using Diacritics.Extensions;
|
||||
using Emby.Server.Implementations.Playlists;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
|
|
|
@ -5,9 +5,9 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Data.Entities;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Dto;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
using MediaBrowser.Controller.Library;
|
||||
|
|
|
@ -3,7 +3,7 @@ using System.Collections.Generic;
|
|||
using System.Globalization;
|
||||
using System.IO;
|
||||
using BlurHashSharp.SkiaSharp;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Controller.Drawing;
|
||||
|
|
|
@ -8,9 +8,9 @@ using System.Linq;
|
|||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Data.Entities;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Providers;
|
||||
using MediaBrowser.Model.Entities;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
|
|
@ -5,8 +5,8 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace MediaBrowser.Controller.Entities.Audio
|
||||
|
|
|
@ -11,7 +11,6 @@ using System.Text;
|
|||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Data.Entities;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
|
|
|
@ -5,8 +5,8 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace MediaBrowser.Controller.Entities
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Providers;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace MediaBrowser.Controller.Entities
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
|
||||
namespace MediaBrowser.Controller.Library
|
||||
{
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Diacritics" Version="3.3.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
|
||||
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
|
||||
|
|
|
@ -8,7 +8,7 @@ using System.Linq;
|
|||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
using MediaBrowser.Controller.Entities.Audio;
|
||||
|
|
|
@ -13,7 +13,7 @@ using System.Text;
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using Diacritics.Extensions;
|
||||
using Jellyfin.Extensions;
|
||||
using MediaBrowser.Controller.Entities.Audio;
|
||||
using MediaBrowser.Controller.Providers;
|
||||
using MediaBrowser.Model.Entities;
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Jellyfin.Extensions
|
||||
{
|
||||
|
@ -7,6 +11,44 @@ namespace Jellyfin.Extensions
|
|||
/// </summary>
|
||||
public static class StringExtensions
|
||||
{
|
||||
// Matches non-conforming unicode chars
|
||||
// https://mnaoumov.wordpress.com/2014/06/14/stripping-invalid-characters-from-utf-16-strings/
|
||||
private static readonly Regex _nonConformingUnicode = new Regex("([\ud800-\udbff](?![\udc00-\udfff]))|((?<![\ud800-\udbff])[\udc00-\udfff])|(\ufffd)");
|
||||
|
||||
/// <summary>
|
||||
/// Removes the diacritics character from the strings.
|
||||
/// </summary>
|
||||
/// <param name="text">The string to act on.</param>
|
||||
/// <returns>The string without diacritics character.</returns>
|
||||
public static string RemoveDiacritics(this string text)
|
||||
{
|
||||
string withDiactritics = _nonConformingUnicode
|
||||
.Replace(text, string.Empty)
|
||||
.Normalize(NormalizationForm.FormD);
|
||||
|
||||
var withoutDiactritics = new StringBuilder();
|
||||
foreach (char c in withDiactritics)
|
||||
{
|
||||
UnicodeCategory uc = CharUnicodeInfo.GetUnicodeCategory(c);
|
||||
if (uc != UnicodeCategory.NonSpacingMark)
|
||||
{
|
||||
withoutDiactritics.Append(c);
|
||||
}
|
||||
}
|
||||
|
||||
return withoutDiactritics.ToString().Normalize(NormalizationForm.FormC);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks wether or not the specified string has diacritics in it.
|
||||
/// </summary>
|
||||
/// <param name="text">The string to check.</param>
|
||||
/// <returns>True if the string has diacritics, false otherwise.</returns>
|
||||
public static bool HasDiacritics(this string text)
|
||||
{
|
||||
return !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts the number of occurrences of [needle] in the string.
|
||||
/// </summary>
|
||||
|
|
|
@ -5,6 +5,38 @@ namespace Jellyfin.Extensions.Tests
|
|||
{
|
||||
public class StringExtensionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("", "")] // Identity edge-case (no diactritics)
|
||||
[InlineData("Indiana Jones", "Indiana Jones")] // Identity (no diactritics)
|
||||
[InlineData("a\ud800b", "ab")] // Invalid UTF-16 char stripping
|
||||
[InlineData("Jön", "Jon")] // Issue #7484
|
||||
[InlineData("Jönssonligan", "Jonssonligan")] // Issue #7484
|
||||
[InlineData("Kieślowski", "Kieslowski")] // Issue #7450
|
||||
[InlineData("Cidadão Kane", "Cidadao Kane")] // Issue #7560
|
||||
[InlineData("운명처럼 널 사랑해", "운명처럼 널 사랑해")] // Issue #6393 (Korean language support)
|
||||
[InlineData("애타는 로맨스", "애타는 로맨스")] // Issue #6393
|
||||
public void RemoveDiacritics_ValidInput_Corrects(string input, string expectedResult)
|
||||
{
|
||||
string result = input.RemoveDiacritics();
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("", false)] // Identity edge-case (no diactritics)
|
||||
[InlineData("Indiana Jones", false)] // Identity (no diactritics)
|
||||
[InlineData("a\ud800b", true)] // Invalid UTF-16 char stripping
|
||||
[InlineData("Jön", true)] // Issue #7484
|
||||
[InlineData("Jönssonligan", true)] // Issue #7484
|
||||
[InlineData("Kieślowski", true)] // Issue #7450
|
||||
[InlineData("Cidadão Kane", true)] // Issue #7560
|
||||
[InlineData("운명처럼 널 사랑해", false)] // Issue #6393 (Korean language support)
|
||||
[InlineData("애타는 로맨스", false)] // Issue #6393
|
||||
public void HasDiacritics_ValidInput_Corrects(string input, bool expectedResult)
|
||||
{
|
||||
bool result = input.HasDiacritics();
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("", '_', 0)]
|
||||
[InlineData("___", '_', 3)]
|
||||
|
|
Loading…
Reference in New Issue
Block a user