jellyfin/Emby.Common.Implementations/TextEncoding/UniversalDetector/Core/SBCSGroupProber.cs

/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is Mozilla Universal charset detector code.
 *
 * The Initial Developer of the Original Code is
 * Netscape Communications Corporation.
 * Portions created by the Initial Developer are Copyright (C) 2001
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *          Shy Shalom <shooshX@gmail.com>
 *          Rudi Pettazzi <rudi.pettazzi@gmail.com> (C# port)
 * 
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

using System;

namespace UniversalDetector.Core
{
    public class SBCSGroupProber : CharsetProber
    {
        private const int PROBERS_NUM = 13;
        private CharsetProber[] probers = new CharsetProber[PROBERS_NUM];        
        private bool[] isActive = new bool[PROBERS_NUM];
        private int bestGuess;
        private int activeNum;
        
        public SBCSGroupProber()
        {
            probers[0] = new SingleByteCharSetProber(new Win1251Model());
            probers[1] = new SingleByteCharSetProber(new Koi8rModel());
            probers[2] = new SingleByteCharSetProber(new Latin5Model());
            probers[3] = new SingleByteCharSetProber(new MacCyrillicModel());
            probers[4] = new SingleByteCharSetProber(new Ibm866Model());
            probers[5] = new SingleByteCharSetProber(new Ibm855Model());
            probers[6] = new SingleByteCharSetProber(new Latin7Model());
            probers[7] = new SingleByteCharSetProber(new Win1253Model());
            probers[8] = new SingleByteCharSetProber(new Latin5BulgarianModel());
            probers[9] = new SingleByteCharSetProber(new Win1251BulgarianModel());
            HebrewProber hebprober = new HebrewProber();
            probers[10] = hebprober;
            // Logical  
            probers[11] = new SingleByteCharSetProber(new Win1255Model(), false, hebprober); 
            // Visual
            probers[12] = new SingleByteCharSetProber(new Win1255Model(), true, hebprober); 
            hebprober.SetModelProbers(probers[11], probers[12]);
            // disable latin2 before latin1 is available, otherwise all latin1 
            // will be detected as latin2 because of their similarity.
            //probers[13] = new SingleByteCharSetProber(new Latin2HungarianModel());
            //probers[14] = new SingleByteCharSetProber(new Win1250HungarianModel());            
            Reset();
        }
  
        public override ProbingState HandleData(byte[] buf, int offset, int len) 
        {
            ProbingState st = ProbingState.NotMe;
            
            //apply filter to original buffer, and we got new buffer back
            //depend on what script it is, we will feed them the new buffer 
            //we got after applying proper filter
            //this is done without any consideration to KeepEnglishLetters
            //of each prober since as of now, there are no probers here which
            //recognize languages with English characters.
            byte[] newBuf = FilterWithoutEnglishLetters(buf, offset, len);
            if (newBuf.Length == 0)
                return state; // Nothing to see here, move on.
            
            for (int i = 0; i < PROBERS_NUM; i++) {
                if (!isActive[i])
                    continue;
                st = probers[i].HandleData(newBuf, 0, newBuf.Length);
                
                if (st == ProbingState.FoundIt) {
                    bestGuess = i;
                    state = ProbingState.FoundIt;
                    break;
                } else if (st == ProbingState.NotMe) {
                    isActive[i] = false;
                    activeNum--;
                    if (activeNum <= 0) {
                        state = ProbingState.NotMe;
                        break;
                    }
                }
            }
            return state;
        }

        public override float GetConfidence()
        {
            float bestConf = 0.0f, cf;
            switch (state) {
            case ProbingState.FoundIt:
                return 0.99f; //sure yes
            case ProbingState.NotMe:
                return 0.01f;  //sure no
            default:
                for (int i = 0; i < PROBERS_NUM; i++)
                {
                    if (!isActive[i])
                        continue;
                    cf = probers[i].GetConfidence();
                    if (bestConf < cf)
                    {
                        bestConf = cf;
                        bestGuess = i;
                    }
                }
                break;
            }
            return bestConf;
        }

        public override void DumpStatus()
        {
            float cf = GetConfidence();
           // Console.WriteLine(" SBCS Group Prober --------begin status");
            for (int i = 0; i < PROBERS_NUM; i++) {
                if (isActive[i])
                    probers[i].DumpStatus();
                //else
                //Console.WriteLine(" inactive: [{0}] (i.e. confidence is too low).", probers[i].GetCharsetName());

            }
            //Console.WriteLine(" SBCS Group found best match [{0}] confidence {1}.",  probers[bestGuess].GetCharsetName(), cf);
        }

        public override void Reset ()
        {
            int activeNum = 0;
            for (int i = 0; i < PROBERS_NUM; i++) {
                if (probers[i] != null) {
                    probers[i].Reset();
                    isActive[i] = true;
                    activeNum++;
                } else {
                    isActive[i] = false;
                }
            }
            bestGuess = -1;
            state = ProbingState.Detecting;
        }

        public override string GetCharsetName()
        {
            //if we have no answer yet
            if (bestGuess == -1) {
                GetConfidence();
                //no charset seems positive
                if (bestGuess == -1)
                    bestGuess = 0;
            }
            return probers[bestGuess].GetCharsetName();
        }

    }
}