jellyfin-server/Emby.Server.Implementations/TextEncoding/TextEncoding.cs

269 lines
8.3 KiB
C#
Raw Normal View History

2017-04-18 05:53:39 +00:00
using System;
using System.Text;
2016-11-01 04:07:12 +00:00
using MediaBrowser.Model.IO;
using Microsoft.Extensions.Logging;
2017-06-17 22:59:17 +00:00
using MediaBrowser.Model.Serialization;
using MediaBrowser.Model.Text;
using NLangDetect.Core;
using UniversalDetector;
2016-10-30 07:11:37 +00:00
namespace Emby.Server.Implementations.TextEncoding
2016-10-30 07:11:37 +00:00
{
2016-11-08 18:44:23 +00:00
public class TextEncoding : ITextEncoding
2016-10-30 07:11:37 +00:00
{
2016-11-01 04:07:12 +00:00
private readonly IFileSystem _fileSystem;
2017-04-18 05:53:39 +00:00
private readonly ILogger _logger;
2017-06-17 22:59:17 +00:00
private IJsonSerializer _json;
2016-11-01 04:07:12 +00:00
2017-06-17 22:59:17 +00:00
public TextEncoding(IFileSystem fileSystem, ILogger logger, IJsonSerializer json)
2016-11-01 04:07:12 +00:00
{
_fileSystem = fileSystem;
2017-04-18 05:53:39 +00:00
_logger = logger;
2017-06-17 22:59:17 +00:00
_json = json;
2016-11-01 04:07:12 +00:00
}
2016-11-08 18:44:23 +00:00
public Encoding GetASCIIEncoding()
2016-10-30 07:11:37 +00:00
{
2016-11-08 18:44:23 +00:00
return Encoding.ASCII;
2016-10-30 07:11:37 +00:00
}
2016-11-01 04:07:12 +00:00
2017-09-04 01:24:20 +00:00
private Encoding GetInitialEncoding(byte[] buffer, int count)
2016-11-01 04:07:12 +00:00
{
2017-09-04 01:24:20 +00:00
if (count >= 3)
{
if (buffer[0] == 0xef && buffer[1] == 0xbb && buffer[2] == 0xbf)
return Encoding.UTF8;
}
if (count >= 2)
{
if (buffer[0] == 0xfe && buffer[1] == 0xff)
return Encoding.Unicode;
}
if (count >= 4)
{
if (buffer[0] == 0 && buffer[1] == 0 && buffer[2] == 0xfe && buffer[3] == 0xff)
return Encoding.UTF32;
}
2016-11-01 04:07:12 +00:00
2017-09-04 01:24:20 +00:00
if (count >= 3)
{
if (buffer[0] == 0x2b && buffer[1] == 0x2f && buffer[2] == 0x76)
return Encoding.UTF7;
}
var result = new TextEncodingDetect().DetectEncoding(buffer, count);
2017-04-18 05:53:39 +00:00
switch (result)
{
case TextEncodingDetect.CharacterEncoding.Ansi:
return Encoding.ASCII;
case TextEncodingDetect.CharacterEncoding.Ascii:
return Encoding.ASCII;
case TextEncodingDetect.CharacterEncoding.Utf16BeBom:
return Encoding.UTF32;
case TextEncodingDetect.CharacterEncoding.Utf16BeNoBom:
return Encoding.UTF32;
case TextEncodingDetect.CharacterEncoding.Utf16LeBom:
return Encoding.UTF32;
case TextEncodingDetect.CharacterEncoding.Utf16LeNoBom:
return Encoding.UTF32;
case TextEncodingDetect.CharacterEncoding.Utf8Bom:
return Encoding.UTF8;
case TextEncodingDetect.CharacterEncoding.Utf8Nobom:
return Encoding.UTF8;
default:
return null;
}
}
2017-06-17 22:59:17 +00:00
private bool _langDetectInitialized;
2017-09-04 01:24:20 +00:00
public string GetDetectedEncodingName(byte[] bytes, int count, string language, bool enableLanguageDetection)
2017-04-18 05:53:39 +00:00
{
2017-09-04 01:24:20 +00:00
var index = 0;
var encoding = GetInitialEncoding(bytes, count);
2017-04-18 05:53:39 +00:00
if (encoding != null && encoding.Equals(Encoding.UTF8))
{
return "utf-8";
}
2017-06-18 07:11:55 +00:00
if (string.IsNullOrWhiteSpace(language) && enableLanguageDetection)
2017-06-17 22:59:17 +00:00
{
2017-06-18 07:11:55 +00:00
if (!_langDetectInitialized)
{
_langDetectInitialized = true;
LanguageDetector.Initialize(_json);
}
2017-06-17 22:59:17 +00:00
2017-09-04 01:24:20 +00:00
language = DetectLanguage(bytes, index, count);
2017-06-17 22:59:17 +00:00
if (!string.IsNullOrWhiteSpace(language))
{
_logger.LogDebug("Text language detected as {0}", language);
2017-06-17 22:59:17 +00:00
}
}
2017-09-04 01:24:20 +00:00
var charset = DetectCharset(bytes, index, count, language);
2017-04-18 05:53:39 +00:00
if (!string.IsNullOrWhiteSpace(charset))
{
if (string.Equals(charset, "utf-8", StringComparison.OrdinalIgnoreCase))
{
return "utf-8";
}
if (!string.Equals(charset, "windows-1252", StringComparison.OrdinalIgnoreCase))
{
return charset;
}
}
if (!string.IsNullOrWhiteSpace(language))
{
return GetFileCharacterSetFromLanguage(language);
}
2016-11-03 22:34:16 +00:00
return null;
2016-11-01 04:07:12 +00:00
}
2017-04-18 05:53:39 +00:00
2017-09-04 01:24:20 +00:00
private string DetectLanguage(byte[] bytes, int index, int count)
2017-06-17 22:59:17 +00:00
{
try
{
2017-09-04 01:24:20 +00:00
return LanguageDetector.DetectLanguage(Encoding.UTF8.GetString(bytes, index, count));
2017-06-17 22:59:17 +00:00
}
catch (NLangDetectException ex)
{
}
try
{
2017-09-04 01:24:20 +00:00
return LanguageDetector.DetectLanguage(Encoding.ASCII.GetString(bytes, index, count));
2017-06-17 22:59:17 +00:00
}
catch (NLangDetectException ex)
{
}
try
{
2017-09-04 01:24:20 +00:00
return LanguageDetector.DetectLanguage(Encoding.Unicode.GetString(bytes, index, count));
2017-06-17 22:59:17 +00:00
}
catch (NLangDetectException ex)
{
}
return null;
}
2017-04-18 05:53:39 +00:00
public Encoding GetEncodingFromCharset(string charset)
{
if (string.IsNullOrWhiteSpace(charset))
{
throw new ArgumentNullException("charset");
}
_logger.LogDebug("Getting encoding object for character set: {0}", charset);
2017-04-18 05:53:39 +00:00
try
{
return Encoding.GetEncoding(charset);
}
catch (ArgumentException)
{
charset = charset.Replace("-", string.Empty);
_logger.LogDebug("Getting encoding object for character set: {0}", charset);
2017-04-18 05:53:39 +00:00
return Encoding.GetEncoding(charset);
}
}
2017-09-04 01:24:20 +00:00
public Encoding GetDetectedEncoding(byte[] bytes, int size, string language, bool enableLanguageDetection)
2017-04-18 05:53:39 +00:00
{
2017-09-04 01:24:20 +00:00
var charset = GetDetectedEncodingName(bytes, size, language, enableLanguageDetection);
2017-04-18 05:53:39 +00:00
return GetEncodingFromCharset(charset);
}
private string GetFileCharacterSetFromLanguage(string language)
{
// https://developer.xamarin.com/api/type/System.Text.Encoding/
switch (language.ToLower())
{
2018-09-12 17:26:21 +00:00
case "tha":
return "windows-874";
2017-04-18 05:53:39 +00:00
case "hun":
return "windows-1252";
case "pol":
case "cze":
case "ces":
case "slo":
case "srp":
case "hrv":
case "rum":
case "ron":
2018-09-12 17:26:21 +00:00
case "rom":
2017-04-18 05:53:39 +00:00
case "rup":
2017-06-17 22:59:17 +00:00
return "windows-1250";
// albanian
2017-04-18 05:53:39 +00:00
case "alb":
case "sqi":
return "windows-1250";
2017-06-17 22:59:17 +00:00
// slovak
case "slk":
case "slv":
return "windows-1250";
2017-04-18 05:53:39 +00:00
case "ara":
return "windows-1256";
case "heb":
return "windows-1255";
case "grc":
2017-06-17 22:59:17 +00:00
return "windows-1253";
// greek
2017-04-18 05:53:39 +00:00
case "gre":
2017-06-17 22:59:17 +00:00
case "ell":
2017-04-18 05:53:39 +00:00
return "windows-1253";
case "crh":
case "ota":
case "tur":
return "windows-1254";
2017-06-20 19:38:42 +00:00
// bulgarian
case "bul":
2017-06-04 20:27:57 +00:00
case "bgr":
2017-06-20 19:38:42 +00:00
return "windows-1251";
2017-04-18 05:53:39 +00:00
case "rus":
return "windows-1251";
case "vie":
return "windows-1258";
case "kor":
return "cp949";
default:
return "windows-1252";
}
}
2017-09-04 01:24:20 +00:00
private string DetectCharset(byte[] bytes, int index, int count, string language)
2017-04-18 05:53:39 +00:00
{
var detector = new CharsetDetector();
2017-09-04 01:24:20 +00:00
detector.Feed(bytes, index, count);
2017-04-18 05:53:39 +00:00
detector.DataEnd();
var charset = detector.Charset;
// This is often incorrectly indetected. If this happens, try to use other techniques instead
if (string.Equals("x-mac-cyrillic", charset, StringComparison.OrdinalIgnoreCase))
{
if (!string.IsNullOrWhiteSpace(language))
{
return null;
}
}
return charset;
}
2016-10-30 07:11:37 +00:00
}
}