extensions/universalchardet/src/base/nsSBCharSetProber.h

   1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* ***** BEGIN LICENSE BLOCK *****
   3  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4  *
   5  * The contents of this file are subject to the Mozilla Public License Version
   6  * 1.1 (the "License"); you may not use this file except in compliance with
   7  * the License. You may obtain a copy of the License at
   8  * http://www.mozilla.org/MPL/
   9  *
  10  * Software distributed under the License is distributed on an "AS IS" basis,
  11  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12  * for the specific language governing rights and limitations under the
  13  * License.
  14  *
  15  * The Original Code is Mozilla Universal charset detector code.
  16  *
  17  * The Initial Developer of the Original Code is
  18  * Netscape Communications Corporation.
  19  * Portions created by the Initial Developer are Copyright (C) 2001
  20  * the Initial Developer. All Rights Reserved.
  21  *
  22  * Contributor(s):
  23  *          Shy Shalom <shooshX@gmail.com>
  24  *
  25  * Alternatively, the contents of this file may be used under the terms of
  26  * either the GNU General Public License Version 2 or later (the "GPL"), or
  27  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28  * in which case the provisions of the GPL or the LGPL are applicable instead
  29  * of those above. If you wish to allow use of your version of this file only
  30  * under the terms of either the GPL or the LGPL, and not to allow others to
  31  * use your version of this file under the terms of the MPL, indicate your
  32  * decision by deleting the provisions above and replace them with the notice
  33  * and other provisions required by the GPL or the LGPL. If you do not delete
  34  * the provisions above, a recipient may use your version of this file under
  35  * the terms of any one of the MPL, the GPL or the LGPL.
  36  *
  37  * ***** END LICENSE BLOCK ***** */
  38 #ifndef nsSingleByteCharSetProber_h__
  39 #define nsSingleByteCharSetProber_h__
  40
  41 #include "nsCharSetProber.h"
  42
  43 #define SAMPLE_SIZE 64
  44 #define SB_ENOUGH_REL_THRESHOLD  1024
  45 #define POSITIVE_SHORTCUT_THRESHOLD  (float)0.95
  46 #define NEGATIVE_SHORTCUT_THRESHOLD  (float)0.05
  47 #define SYMBOL_CAT_ORDER  250
  48 #define NUMBER_OF_SEQ_CAT 4
  49 #define POSITIVE_CAT   (NUMBER_OF_SEQ_CAT-1)
  50 #define NEGATIVE_CAT   0
  51
  52 typedef struct
  53 {
  54   unsigned char *charToOrderMap;    // [256] table use to find a char's order
  55   char *precedenceMatrix;           // [SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency
  56   float  mTypicalPositiveRatio;     // = freqSeqs / totalSeqs
  57   PRBool keepEnglishLetter;         // says if this script contains English characters (not implemented)
  58   const char* charsetName;
  59 } SequenceModel;
  60
  61
  62 class nsSingleByteCharSetProber : public nsCharSetProber{
  63 public:
  64   nsSingleByteCharSetProber(SequenceModel *model)
  65     :mModel(model), mReversed(PR_FALSE), mNameProber(0) { Reset(); }
  66   nsSingleByteCharSetProber(SequenceModel *model, PRBool reversed, nsCharSetProber* nameProber)
  67     :mModel(model), mReversed(reversed), mNameProber(nameProber) { Reset(); }
  68
  69   virtual const char* GetCharSetName();
  70   virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
  71   virtual nsProbingState GetState(void) {return mState;}
  72   virtual void      Reset(void);
  73   virtual float     GetConfidence(void);
  74   virtual void      SetOpion() {}
  75
  76   // This feature is not implemented yet. any current language model
  77   // contain this parameter as PR_FALSE. No one is looking at this
  78   // parameter or calling this method.
  79   // Moreover, the nsSBCSGroupProber which calls the HandleData of this
  80   // prober has a hard-coded call to FilterWithoutEnglishLetters which gets rid
  81   // of the English letters.
  82   PRBool KeepEnglishLetters() {return mModel->keepEnglishLetter;} // (not implemented)
  83
  84 #ifdef DEBUG_chardet
  85   virtual void  DumpStatus();
  86 #endif
  87
  88 protected:
  89   nsProbingState mState;
  90   const SequenceModel *mModel;
  91   const PRBool mReversed; // PR_TRUE if we need to reverse every pair in the model lookup
  92
  93   //char order of last character
  94   unsigned char mLastOrder;
  95
  96   PRUint32 mTotalSeqs;
  97   PRUint32 mSeqCounters[NUMBER_OF_SEQ_CAT];
  98
  99   PRUint32 mTotalChar;
 100   //characters that fall in our sampling range
 101   PRUint32 mFreqChar;
 102
 103   // Optional auxiliary prober for name decision. created and destroyed by the GroupProber
 104   nsCharSetProber* mNameProber;
 105
 106 };
 107
 108
 109 extern SequenceModel Koi8rModel;
 110 extern SequenceModel Win1251Model;
 111 extern SequenceModel Latin5Model;
 112 extern SequenceModel MacCyrillicModel;
 113 extern SequenceModel Ibm866Model;
 114 extern SequenceModel Ibm855Model;
 115 extern SequenceModel Latin7Model;
 116 extern SequenceModel Win1253Model;
 117 extern SequenceModel Latin5BulgarianModel;
 118 extern SequenceModel Win1251BulgarianModel;
 119 extern SequenceModel Latin2HungarianModel;
 120 extern SequenceModel Win1250HungarianModel;
 121 extern SequenceModel Win1255Model;
 122
 123 #endif /* nsSingleByteCharSetProber_h__ */
 124