From 0e7cbb04651bd9b65668ca1635a4625640639560 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Sat, 17 Jun 2017 18:59:17 -0400 Subject: [PATCH 1/2] add subtitle language detection --- .../Emby.Common.Implementations.csproj | 72 ++++ .../IO/SharpCifs/Util/Sharpen/Extensions.cs | 18 - .../TextEncoding/NLangDetect/Detector.cs | 371 +++++++++++++++++ .../NLangDetect/DetectorFactory.cs | 127 ++++++ .../TextEncoding/NLangDetect/ErrorCode.cs | 15 + .../NLangDetect/Extensions/CharExtensions.cs | 374 ++++++++++++++++++ .../Extensions/RandomExtensions.cs | 51 +++ .../Extensions/StringExtensions.cs | 25 ++ .../NLangDetect/Extensions/UnicodeBlock.cs | 131 ++++++ .../TextEncoding/NLangDetect/GenProfile.cs | 67 ++++ .../NLangDetect/InternalException.cs | 22 ++ .../TextEncoding/NLangDetect/Language.cs | 45 +++ .../NLangDetect/LanguageDetector.cs | 37 ++ .../NLangDetect/NLangDetectException.cs | 23 ++ .../TextEncoding/NLangDetect/ProbVector.cs | 35 ++ .../TextEncoding/NLangDetect/Profiles/afr | 1 + .../TextEncoding/NLangDetect/Profiles/ara | 1 + .../TextEncoding/NLangDetect/Profiles/ben | 1 + .../TextEncoding/NLangDetect/Profiles/bul | 1 + .../TextEncoding/NLangDetect/Profiles/ces | 1 + .../TextEncoding/NLangDetect/Profiles/dan | 1 + .../TextEncoding/NLangDetect/Profiles/deu | 1 + .../TextEncoding/NLangDetect/Profiles/ell | 1 + .../TextEncoding/NLangDetect/Profiles/eng | 1 + .../TextEncoding/NLangDetect/Profiles/est | 1 + .../TextEncoding/NLangDetect/Profiles/fas | 1 + .../TextEncoding/NLangDetect/Profiles/fin | 1 + .../TextEncoding/NLangDetect/Profiles/fra | 1 + .../TextEncoding/NLangDetect/Profiles/guj | 1 + .../TextEncoding/NLangDetect/Profiles/heb | 1 + .../TextEncoding/NLangDetect/Profiles/hin | 1 + .../TextEncoding/NLangDetect/Profiles/hrv | 1 + .../TextEncoding/NLangDetect/Profiles/hun | 1 + .../TextEncoding/NLangDetect/Profiles/ind | 1 + .../TextEncoding/NLangDetect/Profiles/ita | 1 + .../TextEncoding/NLangDetect/Profiles/jpn | 1 + .../TextEncoding/NLangDetect/Profiles/kan | 1 + .../TextEncoding/NLangDetect/Profiles/kor | 1 + .../TextEncoding/NLangDetect/Profiles/lav | 1 + .../TextEncoding/NLangDetect/Profiles/lit | 1 + .../TextEncoding/NLangDetect/Profiles/mal | 1 + .../TextEncoding/NLangDetect/Profiles/mar | 1 + .../TextEncoding/NLangDetect/Profiles/mkd | 1 + .../TextEncoding/NLangDetect/Profiles/nep | 1 + .../TextEncoding/NLangDetect/Profiles/nld | 1 + .../TextEncoding/NLangDetect/Profiles/nor | 1 + .../TextEncoding/NLangDetect/Profiles/pan | 1 + .../TextEncoding/NLangDetect/Profiles/pol | 1 + .../TextEncoding/NLangDetect/Profiles/por | 1 + .../TextEncoding/NLangDetect/Profiles/ron | 1 + .../TextEncoding/NLangDetect/Profiles/rus | 1 + .../TextEncoding/NLangDetect/Profiles/slk | 1 + .../TextEncoding/NLangDetect/Profiles/slv | 1 + .../TextEncoding/NLangDetect/Profiles/som | 1 + .../TextEncoding/NLangDetect/Profiles/spa | 1 + .../TextEncoding/NLangDetect/Profiles/sqi | 1 + .../TextEncoding/NLangDetect/Profiles/swa | 1 + .../TextEncoding/NLangDetect/Profiles/swe | 1 + .../TextEncoding/NLangDetect/Profiles/tam | 1 + .../TextEncoding/NLangDetect/Profiles/tel | 1 + .../TextEncoding/NLangDetect/Profiles/tgl | 1 + .../TextEncoding/NLangDetect/Profiles/tha | 1 + .../TextEncoding/NLangDetect/Profiles/tur | 1 + .../TextEncoding/NLangDetect/Profiles/ukr | 1 + .../TextEncoding/NLangDetect/Profiles/urd | 1 + .../TextEncoding/NLangDetect/Profiles/vie | 1 + .../TextEncoding/NLangDetect/Profiles/zh-cn | 1 + .../TextEncoding/NLangDetect/Profiles/zh-tw | 1 + .../NLangDetect/Utils/LangProfile.cs | 118 ++++++ .../NLangDetect/Utils/Messages.cs | 91 +++++ .../TextEncoding/NLangDetect/Utils/NGram.cs | 330 ++++++++++++++++ .../NLangDetect/Utils/TagExtractor.cs | 76 ++++ .../NLangDetect/Utils/messages.properties | 128 ++++++ .../TextEncoding/TextEncoding.cs | 63 ++- Emby.Server.Core/ApplicationHost.cs | 2 +- .../UserLibrary/BaseItemsRequest.cs | 8 +- 76 files changed, 2256 insertions(+), 26 deletions(-) create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Detector.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/DetectorFactory.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/ErrorCode.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Extensions/CharExtensions.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Extensions/RandomExtensions.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Extensions/StringExtensions.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Extensions/UnicodeBlock.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/GenProfile.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/InternalException.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Language.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/LanguageDetector.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/NLangDetectException.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/ProbVector.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/afr create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ara create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ben create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/bul create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ces create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/dan create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/deu create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ell create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/eng create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/est create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/fas create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/fin create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/fra create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/guj create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/heb create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/hin create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/hrv create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/hun create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ind create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ita create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/jpn create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/kan create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/kor create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/lav create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/lit create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/mal create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/mar create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/mkd create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/nep create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/nld create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/nor create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/pan create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/pol create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/por create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ron create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/rus create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/slk create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/slv create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/som create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/spa create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/sqi create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/swa create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/swe create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/tam create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/tel create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/tgl create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/tha create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/tur create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/ukr create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/urd create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/vie create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/zh-cn create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Profiles/zh-tw create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Utils/LangProfile.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Utils/Messages.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Utils/NGram.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Utils/TagExtractor.cs create mode 100644 Emby.Common.Implementations/TextEncoding/NLangDetect/Utils/messages.properties diff --git a/Emby.Common.Implementations/Emby.Common.Implementations.csproj b/Emby.Common.Implementations/Emby.Common.Implementations.csproj index 50d0d05ec..6efc37603 100644 --- a/Emby.Common.Implementations/Emby.Common.Implementations.csproj +++ b/Emby.Common.Implementations/Emby.Common.Implementations.csproj @@ -317,6 +317,23 @@ + + + + + + + + + + + + + + + + + @@ -368,7 +385,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +