/* ***** BEGIN LICENSE BLOCK ***** * Version: MPL 1.1/GPL 2.0/LGPL 2.1 * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License * for the specific language governing rights and limitations under the * License. * * The Original Code is Mozilla Universal charset detector code. * * The Initial Developer of the Original Code is * Netscape Communications Corporation. * Portions created by the Initial Developer are Copyright (C) 2001 * the Initial Developer. All Rights Reserved. * * Contributor(s): * Shy Shalom * Rudi Pettazzi (C# port) * * Alternatively, the contents of this file may be used under the terms of * either the GNU General Public License Version 2 or later (the "GPL"), or * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), * in which case the provisions of the GPL or the LGPL are applicable instead * of those above. If you wish to allow use of your version of this file only * under the terms of either the GPL or the LGPL, and not to allow others to * use your version of this file under the terms of the MPL, indicate your * decision by deleting the provisions above and replace them with the notice * and other provisions required by the GPL or the LGPL. If you do not delete * the provisions above, a recipient may use your version of this file under * the terms of any one of the MPL, the GPL or the LGPL. * * ***** END LICENSE BLOCK ***** */ namespace UniversalDetector.Core { public class UTF8SMModel : SMModel { private readonly static int[] UTF8_cls = { BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17 BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 40 - 47 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 48 - 4f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 50 - 57 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 58 - 5f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 60 - 67 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 68 - 6f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 70 - 77 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 78 - 7f BitPackage.Pack4bits(2,2,2,2,3,3,3,3), // 80 - 87 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 88 - 8f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 90 - 97 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 98 - 9f BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // a0 - a7 BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // a8 - af BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // b0 - b7 BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // b8 - bf BitPackage.Pack4bits(0,0,6,6,6,6,6,6), // c0 - c7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c8 - cf BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d0 - d7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d8 - df BitPackage.Pack4bits(7,8,8,8,8,8,8,8), // e0 - e7 BitPackage.Pack4bits(8,8,8,8,8,9,8,8), // e8 - ef BitPackage.Pack4bits(10,11,11,11,11,11,11,11), // f0 - f7 BitPackage.Pack4bits(12,13,13,13,14,15,0,0) // f8 - ff }; private readonly static int[] UTF8_st = { BitPackage.Pack4bits(ERROR,START,ERROR,ERROR,ERROR,ERROR, 12, 10),//00-07 BitPackage.Pack4bits( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//10-17 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//20-27 BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//28-2f BitPackage.Pack4bits(ERROR,ERROR, 5, 5, 5, 5,ERROR,ERROR),//30-37 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//38-3f BitPackage.Pack4bits(ERROR,ERROR,ERROR, 5, 5, 5,ERROR,ERROR),//40-47 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//48-4f BitPackage.Pack4bits(ERROR,ERROR, 7, 7, 7, 7,ERROR,ERROR),//50-57 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//58-5f BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR, 7, 7,ERROR,ERROR),//60-67 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//68-6f BitPackage.Pack4bits(ERROR,ERROR, 9, 9, 9, 9,ERROR,ERROR),//70-77 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//78-7f BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR, 9,ERROR,ERROR),//80-87 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//88-8f BitPackage.Pack4bits(ERROR,ERROR, 12, 12, 12, 12,ERROR,ERROR),//90-97 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//98-9f BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR, 12,ERROR,ERROR),//a0-a7 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//a8-af BitPackage.Pack4bits(ERROR,ERROR, 12, 12, 12,ERROR,ERROR,ERROR),//b0-b7 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//b8-bf BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,ERROR,ERROR),//c0-c7 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR) //c8-cf }; private readonly static int[] UTF8CharLenTable = {0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6 }; public UTF8SMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UTF8_cls), 16, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UTF8_st), UTF8CharLenTable, "UTF-8") { } } public class GB18030SMModel : SMModel { private readonly static int[] GB18030_cls = { BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17 BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 30 - 37 BitPackage.Pack4bits(3,3,1,1,1,1,1,1), // 38 - 3f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77 BitPackage.Pack4bits(2,2,2,2,2,2,2,4), // 78 - 7f BitPackage.Pack4bits(5,6,6,6,6,6,6,6), // 80 - 87 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 88 - 8f BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 90 - 97 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // 98 - 9f BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // a0 - a7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // a8 - af BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // b0 - b7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // b8 - bf BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c0 - c7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // c8 - cf BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d0 - d7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // d8 - df BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // e0 - e7 BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // e8 - ef BitPackage.Pack4bits(6,6,6,6,6,6,6,6), // f0 - f7 BitPackage.Pack4bits(6,6,6,6,6,6,6,0) // f8 - ff }; private readonly static int[] GB18030_st = { BitPackage.Pack4bits(ERROR,START,START,START,START,START, 3,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START),//10-17 BitPackage.Pack4bits( 4,ERROR,START,START,ERROR,ERROR,ERROR,ERROR),//18-1f BitPackage.Pack4bits(ERROR,ERROR, 5,ERROR,ERROR,ERROR,ITSME,ERROR),//20-27 BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,START,START) //28-2f }; // To be accurate, the length of class 6 can be either 2 or 4. // But it is not necessary to discriminate between the two since // it is used for frequency analysis only, and we are validating // each code range there as well. So it is safe to set it to be // 2 here. private readonly static int[] GB18030CharLenTable = {0, 1, 1, 1, 1, 1, 2}; public GB18030SMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, GB18030_cls), 7, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, GB18030_st), GB18030CharLenTable, "GB18030") { } } public class BIG5SMModel : SMModel { private readonly static int[] BIG5_cls = { BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17 BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77 BitPackage.Pack4bits(2,2,2,2,2,2,2,1), // 78 - 7f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 80 - 87 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 88 - 8f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 90 - 97 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 98 - 9f BitPackage.Pack4bits(4,3,3,3,3,3,3,3), // a0 - a7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // a8 - af BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // b0 - b7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // b8 - bf BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c0 - c7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c8 - cf BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d0 - d7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d8 - df BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e8 - ef BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // f0 - f7 BitPackage.Pack4bits(3,3,3,3,3,3,3,0) // f8 - ff }; private readonly static int[] BIG5_st = { BitPackage.Pack4bits(ERROR,START,START, 3,ERROR,ERROR,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ITSME,ITSME,ITSME,ITSME,ITSME,ERROR),//08-0f BitPackage.Pack4bits(ERROR,START,START,START,START,START,START,START) //10-17 }; private readonly static int[] BIG5CharLenTable = {0, 1, 1, 2, 0}; public BIG5SMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, BIG5_cls), 5, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, BIG5_st), BIG5CharLenTable, "Big5") { } } public class EUCJPSMModel : SMModel { private readonly static int[] EUCJP_cls = { //BitPacket.Pack4bits(5,4,4,4,4,4,4,4), // 00 - 07 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 00 - 07 BitPackage.Pack4bits(4,4,4,4,4,4,5,5), // 08 - 0f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 10 - 17 BitPackage.Pack4bits(4,4,4,5,4,4,4,4), // 18 - 1f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 20 - 27 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 28 - 2f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 30 - 37 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 38 - 3f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 40 - 47 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 48 - 4f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 50 - 57 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 58 - 5f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 60 - 67 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 68 - 6f BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 70 - 77 BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // 78 - 7f BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 80 - 87 BitPackage.Pack4bits(5,5,5,5,5,5,1,3), // 88 - 8f BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 90 - 97 BitPackage.Pack4bits(5,5,5,5,5,5,5,5), // 98 - 9f BitPackage.Pack4bits(5,2,2,2,2,2,2,2), // a0 - a7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a8 - af BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c8 - cf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7 BitPackage.Pack4bits(0,0,0,0,0,0,0,5) // f8 - ff }; private readonly static int[] EUCJP_st = { BitPackage.Pack4bits( 3, 4, 3, 5,START,ERROR,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME,START,ERROR,START,ERROR,ERROR,ERROR),//10-17 BitPackage.Pack4bits(ERROR,ERROR,START,ERROR,ERROR,ERROR, 3,ERROR),//18-1f BitPackage.Pack4bits( 3,ERROR,ERROR,ERROR,START,START,START,START) //20-27 }; private readonly static int[] EUCJPCharLenTable = { 2, 2, 2, 3, 1, 0 }; public EUCJPSMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCJP_cls), 6, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCJP_st), EUCJPCharLenTable, "EUC-JP") { } } public class EUCKRSMModel : SMModel { private readonly static int[] EUCKR_cls = { //BitPacket.Pack4bits(0,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17 BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 40 - 47 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 48 - 4f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 50 - 57 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 58 - 5f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 60 - 67 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 68 - 6f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 70 - 77 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 78 - 7f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f BitPackage.Pack4bits(0,2,2,2,2,2,2,2), // a0 - a7 BitPackage.Pack4bits(2,2,2,2,2,3,3,3), // a8 - af BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7 BitPackage.Pack4bits(2,3,2,2,2,2,2,2), // c8 - cf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // e0 - e7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // e8 - ef BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // f0 - f7 BitPackage.Pack4bits(2,2,2,2,2,2,2,0) // f8 - ff }; private readonly static int[] EUCKR_st = { BitPackage.Pack4bits(ERROR,START, 3,ERROR,ERROR,ERROR,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START,START) //08-0f }; private readonly static int[] EUCKRCharLenTable = { 0, 1, 2, 0 }; public EUCKRSMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCKR_cls), 4, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCKR_st), EUCKRCharLenTable, "EUC-KR") { } } public class EUCTWSMModel : SMModel { private readonly static int[] EUCTW_cls = { BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 00 - 07 BitPackage.Pack4bits(2,2,2,2,2,2,0,0), // 08 - 0f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 10 - 17 BitPackage.Pack4bits(2,2,2,0,2,2,2,2), // 18 - 1f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 20 - 27 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 28 - 2f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 30 - 37 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 38 - 3f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 78 - 7f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87 BitPackage.Pack4bits(0,0,0,0,0,0,6,0), // 88 - 8f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f BitPackage.Pack4bits(0,3,4,4,4,4,4,4), // a0 - a7 BitPackage.Pack4bits(5,5,1,1,1,1,1,1), // a8 - af BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // b0 - b7 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // b8 - bf BitPackage.Pack4bits(1,1,3,1,3,3,3,3), // c0 - c7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // c8 - cf BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d0 - d7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // d8 - df BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e8 - ef BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // f0 - f7 BitPackage.Pack4bits(3,3,3,3,3,3,3,0) // f8 - ff }; private readonly static int[] EUCTW_st = { BitPackage.Pack4bits(ERROR,ERROR,START, 3, 3, 3, 4,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,START,ERROR),//10-17 BitPackage.Pack4bits(START,START,START,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f BitPackage.Pack4bits( 5,ERROR,ERROR,ERROR,START,ERROR,START,START),//20-27 BitPackage.Pack4bits(START,ERROR,START,START,START,START,START,START) //28-2f }; private readonly static int[] EUCTWCharLenTable = { 0, 0, 1, 2, 2, 2, 3 }; public EUCTWSMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCTW_cls), 7, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, EUCTW_st), EUCTWCharLenTable, "EUC-TW") { } } public class SJISSMModel : SMModel { private readonly static int[] SJIS_cls = { //BitPacket.Pack4bits(0,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07 BitPackage.Pack4bits(1,1,1,1,1,1,0,0), // 08 - 0f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 10 - 17 BitPackage.Pack4bits(1,1,1,0,1,1,1,1), // 18 - 1f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 20 - 27 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 28 - 2f BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 30 - 37 BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 38 - 3f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 40 - 47 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 48 - 4f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 50 - 57 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 58 - 5f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 60 - 67 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 68 - 6f BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 70 - 77 BitPackage.Pack4bits(2,2,2,2,2,2,2,1), // 78 - 7f BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 80 - 87 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 88 - 8f BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 90 - 97 BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // 98 - 9f //0xa0 is illegal in sjis encoding, but some pages does //contain such byte. We need to be more error forgiven. BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a0 - a7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // a8 - af BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b0 - b7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // b8 - bf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c0 - c7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // c8 - cf BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d0 - d7 BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // d8 - df BitPackage.Pack4bits(3,3,3,3,3,3,3,3), // e0 - e7 BitPackage.Pack4bits(3,3,3,3,3,4,4,4), // e8 - ef BitPackage.Pack4bits(4,4,4,4,4,4,4,4), // f0 - f7 BitPackage.Pack4bits(4,4,4,4,4,0,0,0) // f8 - ff }; private readonly static int[] SJIS_st = { BitPackage.Pack4bits(ERROR,START,START, 3,ERROR,ERROR,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME,ERROR,ERROR,START,START,START,START) //10-17 }; private readonly static int[] SJISCharLenTable = { 0, 1, 1, 2, 0, 0 }; public SJISSMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, SJIS_cls), 6, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, SJIS_st), SJISCharLenTable, "Shift_JIS") { } } public class UCS2BESMModel : SMModel { private readonly static int[] UCS2BE_cls = { BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 00 - 07 BitPackage.Pack4bits(0,0,1,0,0,2,0,0), // 08 - 0f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 10 - 17 BitPackage.Pack4bits(0,0,0,3,0,0,0,0), // 18 - 1f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 20 - 27 BitPackage.Pack4bits(0,3,3,3,3,3,0,0), // 28 - 2f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 30 - 37 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 38 - 3f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 40 - 47 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 48 - 4f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 50 - 57 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 58 - 5f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 60 - 67 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 68 - 6f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 70 - 77 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 78 - 7f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a0 - a7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a8 - af BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b0 - b7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b8 - bf BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c0 - c7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c8 - cf BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d0 - d7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d8 - df BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7 BitPackage.Pack4bits(0,0,0,0,0,0,4,5) // f8 - ff }; private readonly static int[] UCS2BE_st = { BitPackage.Pack4bits( 5, 7, 7,ERROR, 4, 3,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME, 6, 6, 6, 6,ERROR,ERROR),//10-17 BitPackage.Pack4bits( 6, 6, 6, 6, 6,ITSME, 6, 6),//18-1f BitPackage.Pack4bits( 6, 6, 6, 6, 5, 7, 7,ERROR),//20-27 BitPackage.Pack4bits( 5, 8, 6, 6,ERROR, 6, 6, 6),//28-2f BitPackage.Pack4bits( 6, 6, 6, 6,ERROR,ERROR,START,START) //30-37 }; private readonly static int[] UCS2BECharLenTable = { 2, 2, 2, 0, 2, 2 }; public UCS2BESMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UCS2BE_cls), 6, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UCS2BE_st), UCS2BECharLenTable, "UTF-16BE") { } } public class UCS2LESMModel : SMModel { private readonly static int[] UCS2LE_cls = { BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 00 - 07 BitPackage.Pack4bits(0,0,1,0,0,2,0,0), // 08 - 0f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 10 - 17 BitPackage.Pack4bits(0,0,0,3,0,0,0,0), // 18 - 1f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 20 - 27 BitPackage.Pack4bits(0,3,3,3,3,3,0,0), // 28 - 2f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 30 - 37 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 38 - 3f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 40 - 47 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 48 - 4f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 50 - 57 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 58 - 5f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 60 - 67 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 68 - 6f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 70 - 77 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 78 - 7f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 80 - 87 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 88 - 8f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 90 - 97 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // 98 - 9f BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a0 - a7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // a8 - af BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b0 - b7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // b8 - bf BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c0 - c7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // c8 - cf BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d0 - d7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // d8 - df BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e0 - e7 BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // e8 - ef BitPackage.Pack4bits(0,0,0,0,0,0,0,0), // f0 - f7 BitPackage.Pack4bits(0,0,0,0,0,0,4,5) // f8 - ff }; private readonly static int[] UCS2LE_st = { BitPackage.Pack4bits( 6, 6, 7, 6, 4, 3,ERROR,ERROR),//00-07 BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f BitPackage.Pack4bits(ITSME,ITSME, 5, 5, 5,ERROR,ITSME,ERROR),//10-17 BitPackage.Pack4bits( 5, 5, 5,ERROR, 5,ERROR, 6, 6),//18-1f BitPackage.Pack4bits( 7, 6, 8, 8, 5, 5, 5,ERROR),//20-27 BitPackage.Pack4bits( 5, 5, 5,ERROR,ERROR,ERROR, 5, 5),//28-2f BitPackage.Pack4bits( 5, 5, 5,ERROR, 5,ERROR,START,START) //30-37 }; private readonly static int[] UCS2LECharLenTable = { 2, 2, 2, 2, 2, 2 }; public UCS2LESMModel() : base( new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UCS2LE_cls), 6, new BitPackage(BitPackage.INDEX_SHIFT_4BITS, BitPackage.SHIFT_MASK_4BITS, BitPackage.BIT_SHIFT_4BITS, BitPackage.UNIT_MASK_4BITS, UCS2LE_st), UCS2LECharLenTable, "UTF-16LE") { } } }