third_party/cld/encodings/compact_lang_det/compact_lang_det.h

   1 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 // Baybayin (ancient script of the Philippines) is detected as TAGALOG.
   6 // Chu Nom (Vietnamese ancient Han characters) is detected as VIETNAMESE.
   7 // HAITIAN_CREOLE is detected as such.
   8 // NORWEGIAN and NORWEGIAN_N are detected separately (but not robustly)
   9 // PORTUGUESE, PORTUGUESE_P, and PORTUGUESE_B are all detected as PORTUGUESE.
  10 // ROMANIAN-Latin is detected as ROMANIAN; ROMANIAN-Cyrillic as MOLDAVIAN.
  11 // SERBO_CROATIAN, BOSNIAN, CROATIAN, SERBIAN, MONTENEGRIN in the Latin script
  12 //  are all detected as CROATIAN; in the Cyrillic script as SERBIAN.
  13 // Zhuang is detected in the Latin script only.
  14 //
  15 // The Google interface languages X_PIG_LATIN and X_KLINGON are detected in the
  16 //  extended calls ExtDetectLanguageSummary(). BorkBorkBork, ElmerFudd, and
  17 //  Hacker are not detected (too little training data).
  18 //
  19 // UNKNOWN_LANGUAGE is returned if no language's internal reliablity measure
  20 //  is high enough. This happens with non-text input such as the bytes of a
  21 //  JPEG, and also with some text in languages outside the Google Language
  22 //  enum, such as Ilonggo.
  23 //
  24 // The following languages are detected in multiple scripts:
  25 //  AZERBAIJANI (Latin, Cyrillic*, Arabic*)
  26 //  BURMESE (Latin, Myanmar)
  27 //  HAUSA (Latin, Arabic)
  28 //  KASHMIRI (Arabic, Devanagari)
  29 //  KAZAKH (Latin, Cyrillic, Arabic)
  30 //  KURDISH (Latin*, Arabic)
  31 //  KYRGYZ (Cyrillic, Arabic)
  32 //  LIMBU (Devanagari, Limbu)
  33 //  MONGOLIAN (Cyrillic, Mongolian)
  34 //  SANSKRIT (Latin, Devanagari)
  35 //  SINDHI (Arabic, Devanagari)
  36 //  TAGALOG (Latin, Tagalog)
  37 //  TAJIK (Cyrillic, Arabic*)
  38 //  TATAR (Latin, Cyrillic, Arabic)
  39 //  TURKMEN (Latin, Cyrillic, Arabic)
  40 //  UIGHUR (Latin, Cyrillic, Arabic)
  41 //  UZBEK (Latin, Cyrillic, Arabic)
  42 //
  43 // * Due to a shortage of training text, AZERBAIJANI is not currently detected
  44 //   in Arabic or Cyrillic scripts, nor KURDISH in Latin script, nor TAJIK in
  45 //   Arabic script.
  46 //
  47
  48 #ifndef ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_
  49 #define ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_
  50
  51 #include "languages/public/languages.h"
  52 #include "encodings/compact_lang_det/win/cld_utf8statetable.h"
  53
  54 namespace cld {
  55   struct CLDTableSummary;
  56 }  // namespace cld
  57
  58 namespace CompactLangDet {
  59   // Scan interchange-valid UTF-8 bytes and detect most likely language,
  60   // or set of languages.
  61   //
  62   // Design goals:
  63   //   Skip over big stretches of HTML tags
  64   //   Able to return ranges of different languages
  65   //   Relatively small tables and relatively fast processing
  66   //   Thread safe
  67   //
  68   // For HTML documents, tags are skipped, along with <script> ... </script>
  69   // and <style> ... </style> sequences, and entities are expanded.
  70   //
  71   // We distinguish between bytes of the raw input buffer and bytes of non-tag
  72   // text letters. Since tags can be over 50% of the bytes of an HTML Page,
  73   // and are nearly all seven-bit ASCII English, we prefer to distinguish
  74   // language mixture fractions based on just the non-tag text.
  75   //
  76   // Inputs: text and text_length
  77   //  Code skips HTML tags and expands HTML entities, unless
  78   //  is_plain_text is true
  79   // Outputs:
  80   //  language3 is an array of the top 3 languages or UNKNOWN_LANGUAGE
  81   //  percent3 is an array of the text percentages 0..100 of the top 3 languages
  82   //  text_bytes is the amount of non-tag/letters-only text found
  83   //  is_reliable set true if the returned Language is some amount more
  84   //  probable then the second-best Language. Calculation is a complex function
  85   //  of the length of the text and the different-script runs of text.
  86   // Return value: the most likely Language for the majority of the input text
  87   //  Length 0 input returns UNKNOWN_LANGUAGE. Very short indeterminate text
  88   //  defaults to ENGLISH.
  89   //
  90   // The first two versions return ENGLISH instead of UNKNOWN_LANGUAGE, for
  91   // backwards compatibility with LLD.
  92   //
  93   // The third version may return UNKNOWN_LANGUAGE, and also returns extended
  94   // language codes from ext_lang_enc.h
  95   //
  96   // Subsetting: For fast detection over large documents, these routines will
  97   // scan non-tag text of the initial part of a document, then will
  98   // skip 4-16 bytes and subsample text in the rest of the document, up to a
  99   // fixed limit (currently 160KB of non-tag letters).
 100   //
 101
 102   struct DetectionTables {
 103     const cld::CLDTableSummary* quadgram_obj;
 104     const UTF8PropObj* unigram_obj;
 105   };
 106
 107   // Scan interchange-valid UTF-8 bytes and detect most likely language
 108   Language DetectLanguage(const DetectionTables* tables,
 109                           const char* buffer,
 110                           int buffer_length,
 111                           bool is_plain_text,
 112                           bool* is_reliable);
 113
 114   // Scan interchange-valid UTF-8 bytes and detect list of top 3 languages.
 115   // language3[0] is also the return value
 116   Language DetectLanguageSummary(
 117                           const DetectionTables* tables,
 118                           const char* buffer,
 119                           int buffer_length,
 120                           bool is_plain_text,
 121                           Language* language3,
 122                           int* percent3,
 123                           int* text_bytes,
 124                           bool* is_reliable);
 125
 126   // Same as above, with hints supplied
 127   // Scan interchange-valid UTF-8 bytes and detect list of top 3 languages.
 128   // language3[0] is also the return value
 129   Language DetectLanguageSummary(
 130                           const DetectionTables* tables,
 131                           const char* buffer,
 132                           int buffer_length,
 133                           bool is_plain_text,
 134                           const char* tld_hint,       // "id" boosts Indonesian
 135                           int encoding_hint,          // SJS boosts Japanese
 136                           Language language_hint,     // ITALIAN boosts it
 137                           Language* language3,
 138                           int* percent3,
 139                           int* text_bytes,
 140                           bool* is_reliable);
 141
 142   // Scan interchange-valid UTF-8 bytes and detect list of top 3 extended
 143   // languages.
 144   //
 145   // Extended languages are additional Google interface languages and Unicode
 146   // single-language scripts, from ext_lang_enc.h. They are experimental and
 147   // this call may be removed.
 148   //
 149   // language3[0] is also the return value
 150   Language ExtDetectLanguageSummary(
 151                           const DetectionTables* tables,
 152                           const char* buffer,
 153                           int buffer_length,
 154                           bool is_plain_text,
 155                           Language* language3,
 156                           int* percent3,
 157                           int* text_bytes,
 158                           bool* is_reliable);
 159
 160   // Same as above, with hints supplied
 161   // Scan interchange-valid UTF-8 bytes and detect list of top 3 extended
 162   // languages.
 163   //
 164   // Extended languages are additional Google interface languages and Unicode
 165   // single-language scripts, from ext_lang_enc.h. They are experimental and
 166   // this call may be removed.
 167   //
 168   // language3[0] is also the return value
 169   Language ExtDetectLanguageSummary(
 170                           const DetectionTables* tables,
 171                           const char* buffer,
 172                           int buffer_length,
 173                           bool is_plain_text,
 174                           const char* tld_hint,       // "id" boosts Indonesian
 175                           int encoding_hint,          // SJS boosts Japanese
 176                           Language language_hint,     // ITALIAN boosts it
 177                           Language* language3,
 178                           int* percent3,
 179                           int* text_bytes,
 180                           bool* is_reliable);
 181
 182   // Same as above, and also returns internal language scores as a ratio to
 183   // normal score for real text in that language. Scores close to 1.0 indicate
 184   // normal text, while scores far away from 1.0 indicate badly-skewed text or
 185   // gibberish
 186   //
 187   Language ExtDetectLanguageSummary(
 188                           const DetectionTables* tables,
 189                           const char* buffer,
 190                           int buffer_length,
 191                           bool is_plain_text,
 192                           const char* tld_hint,       // "id" boosts Indonesian
 193                           int encoding_hint,          // SJS boosts Japanese
 194                           Language language_hint,     // ITALIAN boosts it
 195                           Language* language3,
 196                           int* percent3,
 197                           double* normalized_score3,
 198                           int* text_bytes,
 199                           bool* is_reliable);
 200
 201   // Return version text string
 202   // String is "code_version - data_scrape_date"
 203   const char* DetectLanguageVersion();
 204 };      // End namespace CompactLangDet
 205
 206 #endif  // ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_