jellyfin/Emby.Common.Implementations/TextEncoding/TextEncodingDetect.cs
2017-04-18 01:53:39 -04:00

415 lines
13 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Emby.Common.Implementations.TextEncoding
{
// Copyright 2015-2016 Jonathan Bennett <jon@autoitscript.com>
//
// https://www.autoitscript.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// <summary>
/// Credit: https://github.com/AutoIt/text-encoding-detect
/// </summary>
public class TextEncodingDetect
{
private readonly byte[] _utf16BeBom =
{
0xFE,
0xFF
};
private readonly byte[] _utf16LeBom =
{
0xFF,
0xFE
};
private readonly byte[] _utf8Bom =
{
0xEF,
0xBB,
0xBF
};
private bool _nullSuggestsBinary = true;
private double _utf16ExpectedNullPercent = 70;
private double _utf16UnexpectedNullPercent = 10;
public enum CharacterEncoding
{
None, // Unknown or binary
Ansi, // 0-255
Ascii, // 0-127
Utf8Bom, // UTF8 with BOM
Utf8Nobom, // UTF8 without BOM
Utf16LeBom, // UTF16 LE with BOM
Utf16LeNoBom, // UTF16 LE without BOM
Utf16BeBom, // UTF16-BE with BOM
Utf16BeNoBom // UTF16-BE without BOM
}
/// <summary>
/// Sets if the presence of nulls in a buffer indicate the buffer is binary data rather than text.
/// </summary>
public bool NullSuggestsBinary
{
set
{
_nullSuggestsBinary = value;
}
}
public double Utf16ExpectedNullPercent
{
set
{
if (value > 0 && value < 100)
{
_utf16ExpectedNullPercent = value;
}
}
}
public double Utf16UnexpectedNullPercent
{
set
{
if (value > 0 && value < 100)
{
_utf16UnexpectedNullPercent = value;
}
}
}
/// <summary>
/// Gets the BOM length for a given Encoding mode.
/// </summary>
/// <param name="encoding"></param>
/// <returns>The BOM length.</returns>
public static int GetBomLengthFromEncodingMode(CharacterEncoding encoding)
{
int length;
switch (encoding)
{
case CharacterEncoding.Utf16BeBom:
case CharacterEncoding.Utf16LeBom:
length = 2;
break;
case CharacterEncoding.Utf8Bom:
length = 3;
break;
default:
length = 0;
break;
}
return length;
}
/// <summary>
/// Checks for a BOM sequence in a byte buffer.
/// </summary>
/// <param name="buffer"></param>
/// <param name="size"></param>
/// <returns>Encoding type or Encoding.None if no BOM.</returns>
public CharacterEncoding CheckBom(byte[] buffer, int size)
{
// Check for BOM
if (size >= 2 && buffer[0] == _utf16LeBom[0] && buffer[1] == _utf16LeBom[1])
{
return CharacterEncoding.Utf16LeBom;
}
if (size >= 2 && buffer[0] == _utf16BeBom[0] && buffer[1] == _utf16BeBom[1])
{
return CharacterEncoding.Utf16BeBom;
}
if (size >= 3 && buffer[0] == _utf8Bom[0] && buffer[1] == _utf8Bom[1] && buffer[2] == _utf8Bom[2])
{
return CharacterEncoding.Utf8Bom;
}
return CharacterEncoding.None;
}
/// <summary>
/// Automatically detects the Encoding type of a given byte buffer.
/// </summary>
/// <param name="buffer">The byte buffer.</param>
/// <param name="size">The size of the byte buffer.</param>
/// <returns>The Encoding type or Encoding.None if unknown.</returns>
public CharacterEncoding DetectEncoding(byte[] buffer, int size)
{
// First check if we have a BOM and return that if so
CharacterEncoding encoding = CheckBom(buffer, size);
if (encoding != CharacterEncoding.None)
{
return encoding;
}
// Now check for valid UTF8
encoding = CheckUtf8(buffer, size);
if (encoding != CharacterEncoding.None)
{
return encoding;
}
// Now try UTF16
encoding = CheckUtf16NewlineChars(buffer, size);
if (encoding != CharacterEncoding.None)
{
return encoding;
}
encoding = CheckUtf16Ascii(buffer, size);
if (encoding != CharacterEncoding.None)
{
return encoding;
}
// ANSI or None (binary) then
if (!DoesContainNulls(buffer, size))
{
return CharacterEncoding.Ansi;
}
// Found a null, return based on the preference in null_suggests_binary_
return _nullSuggestsBinary ? CharacterEncoding.None : CharacterEncoding.Ansi;
}
/// <summary>
/// Checks if a buffer contains text that looks like utf16 by scanning for
/// newline chars that would be present even in non-english text.
/// </summary>
/// <param name="buffer">The byte buffer.</param>
/// <param name="size">The size of the byte buffer.</param>
/// <returns>Encoding.none, Encoding.Utf16LeNoBom or Encoding.Utf16BeNoBom.</returns>
private static CharacterEncoding CheckUtf16NewlineChars(byte[] buffer, int size)
{
if (size < 2)
{
return CharacterEncoding.None;
}
// Reduce size by 1 so we don't need to worry about bounds checking for pairs of bytes
size--;
var leControlChars = 0;
var beControlChars = 0;
uint pos = 0;
while (pos < size)
{
byte ch1 = buffer[pos++];
byte ch2 = buffer[pos++];
if (ch1 == 0)
{
if (ch2 == 0x0a || ch2 == 0x0d)
{
++beControlChars;
}
}
else if (ch2 == 0)
{
if (ch1 == 0x0a || ch1 == 0x0d)
{
++leControlChars;
}
}
// If we are getting both LE and BE control chars then this file is not utf16
if (leControlChars > 0 && beControlChars > 0)
{
return CharacterEncoding.None;
}
}
if (leControlChars > 0)
{
return CharacterEncoding.Utf16LeNoBom;
}
return beControlChars > 0 ? CharacterEncoding.Utf16BeNoBom : CharacterEncoding.None;
}
/// <summary>
/// Checks if a buffer contains any nulls. Used to check for binary vs text data.
/// </summary>
/// <param name="buffer">The byte buffer.</param>
/// <param name="size">The size of the byte buffer.</param>
private static bool DoesContainNulls(byte[] buffer, int size)
{
uint pos = 0;
while (pos < size)
{
if (buffer[pos++] == 0)
{
return true;
}
}
return false;
}
/// <summary>
/// Checks if a buffer contains text that looks like utf16. This is done based
/// on the use of nulls which in ASCII/script like text can be useful to identify.
/// </summary>
/// <param name="buffer">The byte buffer.</param>
/// <param name="size">The size of the byte buffer.</param>
/// <returns>Encoding.none, Encoding.Utf16LeNoBom or Encoding.Utf16BeNoBom.</returns>
private CharacterEncoding CheckUtf16Ascii(byte[] buffer, int size)
{
var numOddNulls = 0;
var numEvenNulls = 0;
// Get even nulls
uint pos = 0;
while (pos < size)
{
if (buffer[pos] == 0)
{
numEvenNulls++;
}
pos += 2;
}
// Get odd nulls
pos = 1;
while (pos < size)
{
if (buffer[pos] == 0)
{
numOddNulls++;
}
pos += 2;
}
double evenNullThreshold = numEvenNulls * 2.0 / size;
double oddNullThreshold = numOddNulls * 2.0 / size;
double expectedNullThreshold = _utf16ExpectedNullPercent / 100.0;
double unexpectedNullThreshold = _utf16UnexpectedNullPercent / 100.0;
// Lots of odd nulls, low number of even nulls
if (evenNullThreshold < unexpectedNullThreshold && oddNullThreshold > expectedNullThreshold)
{
return CharacterEncoding.Utf16LeNoBom;
}
// Lots of even nulls, low number of odd nulls
if (oddNullThreshold < unexpectedNullThreshold && evenNullThreshold > expectedNullThreshold)
{
return CharacterEncoding.Utf16BeNoBom;
}
// Don't know
return CharacterEncoding.None;
}
/// <summary>
/// Checks if a buffer contains valid utf8.
/// </summary>
/// <param name="buffer">The byte buffer.</param>
/// <param name="size">The size of the byte buffer.</param>
/// <returns>
/// Encoding type of Encoding.None (invalid UTF8), Encoding.Utf8NoBom (valid utf8 multibyte strings) or
/// Encoding.ASCII (data in 0.127 range).
/// </returns>
/// <returns>2</returns>
private CharacterEncoding CheckUtf8(byte[] buffer, int size)
{
// UTF8 Valid sequences
// 0xxxxxxx ASCII
// 110xxxxx 10xxxxxx 2-byte
// 1110xxxx 10xxxxxx 10xxxxxx 3-byte
// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 4-byte
//
// Width in UTF8
// Decimal Width
// 0-127 1 byte
// 194-223 2 bytes
// 224-239 3 bytes
// 240-244 4 bytes
//
// Subsequent chars are in the range 128-191
var onlySawAsciiRange = true;
uint pos = 0;
while (pos < size)
{
byte ch = buffer[pos++];
if (ch == 0 && _nullSuggestsBinary)
{
return CharacterEncoding.None;
}
int moreChars;
if (ch <= 127)
{
// 1 byte
moreChars = 0;
}
else if (ch >= 194 && ch <= 223)
{
// 2 Byte
moreChars = 1;
}
else if (ch >= 224 && ch <= 239)
{
// 3 Byte
moreChars = 2;
}
else if (ch >= 240 && ch <= 244)
{
// 4 Byte
moreChars = 3;
}
else
{
return CharacterEncoding.None; // Not utf8
}
// Check secondary chars are in range if we are expecting any
while (moreChars > 0 && pos < size)
{
onlySawAsciiRange = false; // Seen non-ascii chars now
ch = buffer[pos++];
if (ch < 128 || ch > 191)
{
return CharacterEncoding.None; // Not utf8
}
--moreChars;
}
}
// If we get to here then only valid UTF-8 sequences have been processed
// If we only saw chars in the range 0-127 then we can't assume UTF8 (the caller will need to decide)
return onlySawAsciiRange ? CharacterEncoding.Ascii : CharacterEncoding.Utf8Nobom;
}
}
}