?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Don't preload rarely seen large images
[chromium-blink-merge.git] / third_party / cld / encodings / compact_lang_det / cldutil.h
blob7125343d8ded494a4b107029fc9caac2fc06455d
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_
6 #define ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_
7
8 #include <string>
9 #include "encodings/compact_lang_det/ext_lang_enc.h"
10 #include "encodings/compact_lang_det/tote.h"
11 #include "encodings/compact_lang_det/win/cld_basictypes.h"
12 #include "encodings/compact_lang_det/win/cld_commandlineflags.h"
13 #include "encodings/compact_lang_det/win/cld_utf8statetable.h"
14
15 namespace cld {
16
17 // Hash bucket for four-way associative lookup with < 64K buckets
18 // 32 bytes per bucket, 8-byte entries
19 typedef struct {
20 uint32 key[4]; // hashed word to look up
21 uint32 value[4]; // packed three lang numbers and probability subscript
22 } SmallWordProbBucket4;
23
24 // Hash bucket for fouro-way associative lookup with >= 64K buckets
25 // 24 bytes per bucket, 6-byte entries
26 typedef struct {
27 uint16 key[4]; // Half of hashed word to look up; other
28 // half is used to pick the bucket
29 uint32 value[4]; // packed three lang numbers and probability subscript
30 } LargeQuadProbBucket4;
31
32 // Hash bucket for four-way associative lookup, indirect probabilities
33 // 16 bytes per bucket, 4-byte entries
34 typedef struct {
35 uint32 keyvalue[4]; // Upper part of word is hash, lower is indirect prob
36 } IndirectProbBucket4;
37
38
39 // This describes a complete CLD table, consisting of
40 // a main lookup table, an indirect language/probability table, and
41 // three constants.
42 // The main table key is a quadgram, bigram, or longword hash, with
43 // part of the key used to select a bucket modulo kCLDTableSize,
44 // and the rest matched against the key portion of four entries in a bucket,
45 // defined by kCLDTableKeyMask. The remaining bits of an entry, defined
46 // by ~kCLDTableKeyMask, are usually a subscript in the indirect table.
47 //
48 // By using part of the key to select a bucket, those key bits do not need
49 // to be stored in the main table entries, saving space (typically 2 bytes).
50 //
51 // By using an indirect table for lang/prob triples, only the subscript needs
52 // to be stored in the main table entires, saving space (typically 2 bytes).
53 //
54 // Each entry in the indirect table has three languages and three
55 // corresponding probabilities, packed into four bytes.
56 //
57 // The build date constant is included just for version tracking and is not
58 // otherwise used.
59 //
60 // Different-size tables can be linked in for different production
61 // environments. By going indirect through this struct, the runtime code is
62 // insensitive to the actual sizes.
63 //
64 // An empty placeholder table can be described by a table size of 1
65 // bucket, a keymask of 0xffffffff, a degenerate bucket of four no-match
66 // entries, and a degenerate indirect table of one no-languages entry.
67 //
68 //
69 struct CLDTableSummary {
70 const IndirectProbBucket4* kCLDTable;
71 // Each bucket has four entries, part
72 // key and part indirect subscript
73 const uint32* kCLDTableInd; // Each entry is three packed lang/prob
74 const int kCLDTableSize; // Bucket count
75 const int kCLDTableIndSize; // Entries count
76 const int kCLDTableKeyMask; // Mask hash key
77 const int kCLDTableBuildDate; // yyyymmdd
78 };
79
80
81 // Keeps per-character 0-12 language probabilities for CTJKVZ-- in that order.
82 // Chinese ChineseT Japanese Korean Vietnamese Zhuang
83 // (2 bytes unused, for alignment padding and future)
84 typedef struct {
85 uint8 probs[8];
86 } UnigramProbArray;
87
88 // Map 8-bit subscript to CTJKVZ probabilities
89 // Target runtime probabilities for CTJK + VZ
90 // Hand-generated to cover a reasonable range of choices
91 static const int kTargetCTJKVZProbsSize = 242;
92 static const UnigramProbArray kTargetCTJKVZProbs[kTargetCTJKVZProbsSize] = {
93 {{0,0,0,0,0,0,0,0}},
94 {{0,0,0,0,0,12,0,0}},
95 {{0,0,0,0,12,0,0,0}},
96 {{0,0,0,12,0,0,0,0}},
97 {{0,0,12,0,0,0,0,0}},
98 {{0,12,0,0,0,0,0,0}},
99 {{12,0,0,0,0,0,0,0}},
100
101 {{8,0,0,0,4,0,0,0}},
102 {{8,0,0,4,0,0,0,0}},
103 {{8,0,4,0,0,0,0,0}},
104 {{8,4,0,0,0,0,0,0}},
105 {{8,2,0,2,0,0,0,0}},
106 {{0,0,0,0,0,8,0,0}},
107 {{0,4,8,0,0,0,0,0}},
108 {{4,0,0,0,0,8,0,0}},
109 {{0,0,8,0,0,0,0,0}},
110 {{8,2,2,0,0,0,0,0}},
111 {{0,8,4,0,0,0,0,0}},
112 {{8,0,0,0,0,4,0,0}},
113 {{0,8,2,0,0,0,0,0}},
114 {{4,8,0,0,0,0,0,0}},
115 {{2,8,0,2,0,0,0,0}},
116 {{2,2,8,0,0,0,0,0}},
117 {{0,8,0,0,0,0,0,0}},
118 {{0,2,8,0,0,0,0,0}},
119 {{2,8,2,0,0,0,0,0}},
120 {{8,0,0,0,0,0,0,0}},
121 {{2,8,0,0,0,0,0,0}},
122 {{8,2,0,0,0,0,0,0}},
123
124 {{0,6,2,0,2,0,0,0}},
125 {{2,0,0,0,6,0,0,0}},
126 {{4,0,0,0,6,0,0,0}},
127 {{4,6,0,0,4,0,0,0}},
128 {{4,6,2,0,2,0,0,0}},
129 {{4,6,4,0,2,0,0,0}},
130 {{5,4,6,0,0,0,0,0}},
131 {{6,0,0,0,4,0,0,0}},
132 {{6,0,2,0,4,0,0,0}},
133 {{6,0,4,0,4,0,0,0}},
134 {{6,2,0,0,4,0,0,0}},
135 {{6,2,2,0,4,0,0,0}},
136 {{6,2,4,0,2,0,0,0}},
137 {{6,4,0,0,2,0,0,0}},
138 {{6,4,2,0,2,0,0,0}},
139 {{0,0,6,2,0,0,0,0}},
140 {{0,6,2,0,0,2,0,0}},
141 {{2,2,2,0,0,6,0,0}},
142 {{2,2,6,4,0,0,0,0}},
143 {{2,4,0,0,0,6,0,0}},
144 {{2,6,0,4,0,0,0,0}},
145 {{2,6,2,4,0,0,0,0}},
146 {{2,6,4,4,0,0,0,0}},
147 {{4,0,2,0,0,6,0,0}},
148 {{4,2,6,2,0,0,0,0}},
149 {{4,4,2,0,0,6,0,0}},
150 {{4,6,4,0,0,2,0,0}},
151 {{6,0,2,0,0,2,0,0}},
152 {{6,2,0,0,0,2,0,0}},
153 {{6,2,2,0,0,4,0,0}},
154 {{6,2,4,0,0,2,0,0}},
155 {{4,6,2,0,0,4,0,0}},
156 {{6,4,2,0,0,4,0,0}},
157 {{2,0,0,0,0,6,0,0}},
158 {{6,2,0,2,0,0,0,0}},
159 {{2,2,0,0,0,6,0,0}},
160 {{6,2,6,0,0,0,0,0}},
161 {{6,4,2,0,0,2,0,0}},
162 {{6,4,2,2,0,0,0,0}},
163 {{4,6,4,2,0,0,0,0}},
164 {{6,0,2,0,0,4,0,0}},
165 {{6,0,4,0,0,2,0,0}},
166 {{6,0,6,0,0,0,0,0}},
167 {{6,2,2,0,0,0,0,0}},
168 {{6,4,0,0,0,2,0,0}},
169 {{6,4,5,0,0,0,0,0}},
170 {{0,6,0,2,0,0,0,0}},
171 {{0,6,2,2,0,0,0,0}},
172 {{2,6,0,2,0,0,0,0}},
173 {{2,6,2,2,0,0,0,0}},
174 {{4,2,0,0,0,6,0,0}},
175 {{6,4,0,0,0,4,0,0}},
176 {{6,4,0,2,0,0,0,0}},
177 {{6,6,0,2,0,0,0,0}},
178 {{6,0,4,0,0,4,0,0}},
179 {{6,2,0,0,0,4,0,0}},
180 {{6,6,2,2,0,0,0,0}},
181 {{4,6,0,0,0,2,0,0}},
182 {{2,6,6,0,0,0,0,0}},
183 {{4,5,6,0,0,0,0,0}},
184 {{4,6,0,2,0,0,0,0}},
185 {{6,2,0,0,0,6,0,0}},
186 {{0,6,4,2,0,0,0,0}},
187 {{4,0,6,0,0,0,0,0}},
188 {{2,6,4,2,0,0,0,0}},
189 {{4,6,0,0,0,4,0,0}},
190 {{6,2,2,0,0,0,0,0}},
191 {{4,6,2,2,0,0,0,0}},
192 {{4,6,5,0,0,0,0,0}},
193 {{6,0,2,0,0,0,0,0}},
194 {{6,4,4,0,0,0,0,0}},
195 {{4,2,6,0,0,0,0,0}},
196 {{2,0,6,0,0,0,0,0}},
197 {{4,4,0,0,0,6,0,0}},
198 {{4,4,6,0,0,0,0,0}},
199 {{4,6,2,0,0,2,0,0}},
200 {{2,2,6,0,0,0,0,0}},
201 {{2,4,6,0,0,0,0,0}},
202 {{0,6,6,0,0,0,0,0}},
203 {{6,2,4,0,0,0,0,0}},
204 {{0,4,6,0,0,0,0,0}},
205 {{4,0,0,0,0,6,0,0}},
206 {{4,6,4,0,0,0,0,0}},
207 {{6,0,0,0,0,6,0,0}},
208 {{6,0,0,0,0,2,0,0}},
209 {{6,0,4,0,0,0,0,0}},
210 {{6,5,4,0,0,0,0,0}},
211 {{0,2,6,0,0,0,0,0}},
212 {{0,0,6,0,0,0,0,0}},
213 {{6,6,2,0,0,0,0,0}},
214 {{2,6,4,0,0,0,0,0}},
215 {{6,4,2,0,0,0,0,0}},
216 {{2,6,2,0,0,0,0,0}},
217 {{2,6,0,0,0,0,0,0}},
218 {{6,0,0,0,0,4,0,0}},
219 {{6,4,0,0,0,0,0,0}},
220 {{6,6,0,0,0,0,0,0}},
221 {{5,6,4,0,0,0,0,0}},
222 {{0,6,0,0,0,0,0,0}},
223 {{6,2,0,0,0,0,0,0}},
224 {{0,6,2,0,0,0,0,0}},
225 {{4,6,2,0,0,0,0,0}},
226 {{0,6,4,0,0,0,0,0}},
227 {{4,6,0,0,0,0,0,0}},
228 {{6,0,0,0,0,0,0,0}},
229 {{6,6,5,0,0,0,0,0}},
230 {{6,5,6,0,0,0,0,0}},
231 {{5,6,6,0,0,0,0,0}},
232 {{5,5,6,0,0,0,0,0}},
233 {{5,6,5,0,0,0,0,0}},
234 {{6,5,5,0,0,0,0,0}},
235 {{6,6,6,0,0,0,0,0}},
236 {{6,5,0,0,0,0,0,0}},
237 {{6,0,5,0,0,0,0,0}},
238 {{0,6,5,0,0,0,0,0}},
239 {{5,6,0,0,0,0,0,0}},
240 {{5,0,6,0,0,0,0,0}},
241 {{0,5,6,0,0,0,0,0}},
242
243 {{0,0,0,0,4,0,0,0}},
244 {{0,0,0,4,0,0,0,0}},
245 {{2,2,0,0,4,0,0,0}},
246 {{2,2,2,0,4,0,0,0}},
247 {{2,4,0,0,2,0,0,0}},
248 {{2,4,2,0,2,0,0,0}},
249 {{2,4,4,0,2,0,0,0}},
250 {{4,0,2,0,4,0,0,0}},
251 {{4,0,4,0,2,0,0,0}},
252 {{4,2,0,0,2,0,0,0}},
253 {{4,2,2,0,2,0,0,0}},
254 {{4,4,0,0,2,0,0,0}},
255 {{4,4,2,0,2,0,0,0}},
256 {{4,4,4,0,2,0,0,0}},
257 {{0,2,2,4,0,0,0,0}},
258 {{2,2,4,2,0,0,0,0}},
259 {{2,4,4,0,0,2,0,0}},
260 {{2,4,4,2,0,0,0,0}},
261 {{4,0,4,0,0,2,0,0}},
262 {{4,0,4,0,0,4,0,0}},
263 {{4,2,2,4,0,0,0,0}},
264 {{4,4,0,2,0,0,0,0}},
265 {{2,2,0,4,0,0,0,0}},
266 {{2,4,2,2,0,0,0,0}},
267 {{4,4,2,2,0,0,0,0}},
268 {{4,0,4,0,0,0,0,0}},
269 {{4,4,4,0,0,4,0,0}},
270 {{0,4,0,2,0,0,0,0}},
271 {{0,4,2,2,0,0,0,0}},
272 {{4,0,2,0,0,2,0,0}},
273 {{4,2,0,0,0,4,0,0}},
274 {{2,2,2,0,0,4,0,0}},
275 {{4,0,0,2,0,0,0,0}},
276 {{4,4,4,0,0,2,0,0}},
277 {{4,0,0,0,0,4,0,0}},
278 {{4,0,2,0,0,4,0,0}},
279 {{4,2,0,0,0,2,0,0}},
280 {{4,2,2,0,0,2,0,0}},
281 {{2,4,0,2,0,0,0,0}},
282 {{2,2,0,0,0,4,0,0}},
283 {{2,4,0,0,0,4,0,0}},
284 {{2,4,2,0,0,4,0,0}},
285 {{4,2,4,0,0,0,0,0}},
286 {{2,0,4,0,0,0,0,0}},
287 {{4,0,2,0,0,0,0,0}},
288 {{4,4,0,0,0,4,0,0}},
289 {{4,4,2,0,0,4,0,0}},
290 {{0,4,4,0,0,0,0,0}},
291 {{4,4,0,0,0,2,0,0}},
292 {{2,4,0,0,0,2,0,0}},
293 {{2,2,4,0,0,0,0,0}},
294 {{0,2,4,0,0,0,0,0}},
295 {{4,2,2,0,0,0,0,0}},
296 {{2,4,2,0,0,2,0,0}},
297 {{4,4,4,0,0,0,0,0}},
298 {{2,4,4,0,0,0,0,0}},
299 {{0,0,4,0,0,0,0,0}},
300 {{0,4,2,0,0,0,0,0}},
301 {{4,4,2,0,0,2,0,0}},
302 {{2,4,2,0,0,0,0,0}},
303 {{4,2,0,0,0,0,0,0}},
304 {{4,4,0,0,0,0,0,0}},
305 {{4,4,2,0,0,0,0,0}},
306 {{2,4,0,0,0,0,0,0}},
307 {{0,4,0,0,0,0,0,0}},
308 {{4,0,0,0,0,0,0,0}},
309 {{0,0,0,4,4,0,0,0}},
310 {{0,0,4,0,4,0,0,0}},
311 {{0,0,4,4,0,0,0,0}},
312 {{0,4,0,0,4,0,0,0}},
313 {{0,4,0,4,0,0,0,0}},
314 {{4,0,0,0,4,0,0,0}},
315 {{4,0,0,4,0,0,0,0}},
316
317 {{2,0,0,0,0,0,0,0}},
318 {{0,2,0,0,0,0,0,0}},
319 {{0,2,0,2,2,0,0,0}},
320 {{0,2,2,0,2,0,0,0}},
321 {{2,0,0,2,2,0,0,0}},
322 {{2,0,2,0,2,0,0,0}},
323 {{2,0,2,2,0,0,0,0}},
324 {{2,2,0,0,2,0,0,0}},
325 {{2,2,2,2,0,0,0,0}},
326 {{2,2,0,2,0,0,0,0}},
327 {{2,2,0,0,0,0,0,0}},
328 {{0,0,2,0,0,0,0,0}},
329 {{0,2,2,0,0,0,0,0}},
330 {{2,2,2,0,0,0,0,0}},
331 {{0,0,0,2,0,0,0,0}},
332 {{2,0,2,0,0,0,0,0}},
333 {{0,2,0,2,0,0,0,0}},
334 {{0,0,2,2,0,0,0,0}},
335 {{0,2,2,2,0,0,0,0}},
336 };
337
338
339
340
341 // 1 to skip ASCII space, vowels AEIOU aeiou and UTF-8 continuation bytes 80-BF
342 static const uint8 kSkipSpaceVowelContinue[256] = {
343 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
344 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
345 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
346 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
347
348 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
349 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
350 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
351 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
352 };
353
354 // 1 to skip ASCII space, and UTF-8 continuation bytes 80-BF
355 static const uint8 kSkipSpaceContinue[256] = {
356 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
357 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
358 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
359 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
360
361 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
362 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
363 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
364 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
365 };
366
367
368 // If != UNKNOWN, use nilgrams to determine language of this script
369 static const Language kOnlyLanguagePerLScript[] = {
370 ENGLISH, // ULScript_Common, [no words should be in this script]
371 UNKNOWN_LANGUAGE, // ULScript_Latin,
372 //UNKNOWN_LANGUAGE, // ULScript_Greek, Jan 2009: change so we can score quads
373 GREEK, // ULScript_Greek, Mar 2009: change back; do gibberish separately
374 UNKNOWN_LANGUAGE, // ULScript_Cyrillic,
375 ARMENIAN, // ULScript_Armenian,
376 UNKNOWN_LANGUAGE, // ULScript_Hebrew,
377 UNKNOWN_LANGUAGE, // ULScript_Arabic,
378 SYRIAC, // ULScript_Syriac,
379 DHIVEHI, // ULScript_Thaana,
380 UNKNOWN_LANGUAGE, // ULScript_Devanagari,
381 UNKNOWN_LANGUAGE, // ULScript_Bengali,
382 PUNJABI, // ULScript_Gurmukhi,
383 GUJARATI, // ULScript_Gujarati,
384 ORIYA, // ULScript_Oriya,
385 TAMIL, // ULScript_Tamil,
386 TELUGU, // ULScript_Telugu,
387 KANNADA, // ULScript_Kannada,
388 MALAYALAM, // ULScript_Malayalam,
389 SINHALESE, // ULScript_Sinhala,
390 THAI, // ULScript_Thai,
391 LAOTHIAN, // ULScript_Lao,
392 UNKNOWN_LANGUAGE, // ULScript_Tibetan,
393 BURMESE, // ULScript_Myanmar,
394 GEORGIAN, // ULScript_Georgian,
395 UNKNOWN_LANGUAGE, // ULScript_HanCJK,
396 UNKNOWN_LANGUAGE, // ULScript_Ethiopic,
397 CHEROKEE, // ULScript_Cherokee,
398 INUKTITUT, // ULScript_Canadian_Aboriginal,
399 X_OGHAM, // ULScript_Ogham,
400 X_RUNIC, // ULScript_Runic,
401 KHMER, // ULScript_Khmer,
402 MONGOLIAN, // ULScript_Mongolian,
403 X_YI, // ULScript_Yi,
404 X_OLD_ITALIC, // ULScript_Old_Italic,
405 X_GOTHIC, // ULScript_Gothic,
406 X_DESERET, // ULScript_Deseret,
407 ENGLISH, // ULScript_Inherited, [no words should be in this script]
408 TAGALOG, // ULScript_Tagalog,
409 X_HANUNOO, // ULScript_Hanunoo,
410 X_BUHID, // ULScript_Buhid,
411 X_TAGBANWA, // ULScript_Tagbanwa,
412 LIMBU, // ULScript_Limbu,
413 X_TAI_LE, // ULScript_Tai_Le,
414 X_LINEAR_B, // ULScript_Linear_B,
415 X_UGARITIC, // ULScript_Ugaritic,
416 X_SHAVIAN, // ULScript_Shavian,
417 X_OSMANYA, // ULScript_Osmanya,
418 X_CYPRIOT, // ULScript_Cypriot,
419 X_BUGINESE, // ULScript_Buginese,
420 X_COPTIC, // ULScript_Coptic,
421 X_NEW_TAI_LUE, // ULScript_New_Tai_Lue,
422 X_GLAGOLITIC, // ULScript_Glagolitic,
423 X_TIFINAGH, // ULScript_Tifinagh,
424 X_SYLOTI_NAGRI, // ULScript_Syloti_Nagri,
425 X_OLD_PERSIAN, // ULScript_Old_Persian,
426 X_KHAROSHTHI, // ULScript_Kharoshthi,
427 X_BALINESE, // ULScript_Balinese,
428 X_CUNEIFORM, // ULScript_Cuneiform,
429 X_PHOENICIAN, // ULScript_Phoenician,
430 X_PHAGS_PA, // ULScript_Phags_Pa,
431 X_NKO, // ULScript_Nko,
432
433 // Unicode 5.1
434 X_SUDANESE, // ULScript_Sundanese,
435 X_LEPCHA, // ULScript_Lepcha,
436 X_OL_CHIKI, // ULScript_Ol_Chiki,
437 X_VAI, // ULScript_Vai,
438 X_SAURASHTRA, // ULScript_Saurashtra,
439 X_KAYAH_LI, // ULScript_Kayah_Li,
440 X_REJANG, // ULScript_Rejang,
441 X_LYCIAN, // ULScript_Lycian,
442 X_CARIAN, // ULScript_Carian,
443 X_LYDIAN, // ULScript_Lydian,
444 X_CHAM, // ULScript_Cham,
445 };
446
447 COMPILE_ASSERT(arraysize(kOnlyLanguagePerLScript) == ULScript_NUM_SCRIPTS,
448 kOnlyLanguagePerLScript_has_incorrect_length);
449
450
451 // This is, in a sense, the complement of the table above
452 // If != UNKNOWN, determines a default language of this script
453 static const Language kDefaultLanguagePerLScript[] = {
454 UNKNOWN_LANGUAGE, // ULScript_Common, [no words should be in this script]
455 ENGLISH, // ULScript_Latin,
456 UNKNOWN_LANGUAGE, // ULScript_Greek,
457 RUSSIAN, // ULScript_Cyrillic,
458 UNKNOWN_LANGUAGE, // ULScript_Armenian,
459 HEBREW, // ULScript_Hebrew,
460 ARABIC, // ULScript_Arabic,
461 UNKNOWN_LANGUAGE, // ULScript_Syriac,
462 UNKNOWN_LANGUAGE, // ULScript_Thaana,
463 HINDI, // ULScript_Devanagari,
464 BENGALI, // ULScript_Bengali,
465 UNKNOWN_LANGUAGE, // ULScript_Gurmukhi,
466 UNKNOWN_LANGUAGE, // ULScript_Gujarati,
467 UNKNOWN_LANGUAGE, // ULScript_Oriya,
468 UNKNOWN_LANGUAGE, // ULScript_Tamil,
469 UNKNOWN_LANGUAGE, // ULScript_Telugu,
470 UNKNOWN_LANGUAGE, // ULScript_Kannada,
471 UNKNOWN_LANGUAGE, // ULScript_Malayalam,
472 UNKNOWN_LANGUAGE, // ULScript_Sinhala,
473 UNKNOWN_LANGUAGE, // ULScript_Thai,
474 UNKNOWN_LANGUAGE, // ULScript_Lao,
475 TIBETAN, // ULScript_Tibetan,
476 UNKNOWN_LANGUAGE, // ULScript_Myanmar,
477 UNKNOWN_LANGUAGE, // ULScript_Georgian,
478 CHINESE, // ULScript_HanCJK,
479 AMHARIC, // ULScript_Ethiopic,
480 UNKNOWN_LANGUAGE, // ULScript_Cherokee,
481 UNKNOWN_LANGUAGE, // ULScript_Canadian_Aboriginal,
482 UNKNOWN_LANGUAGE, // ULScript_Ogham,
483 UNKNOWN_LANGUAGE, // ULScript_Runic,
484 UNKNOWN_LANGUAGE, // ULScript_Khmer,
485 UNKNOWN_LANGUAGE, // ULScript_Mongolian,
486 UNKNOWN_LANGUAGE, // ULScript_Yi,
487 UNKNOWN_LANGUAGE, // ULScript_Old_Italic,
488 UNKNOWN_LANGUAGE, // ULScript_Gothic,
489 UNKNOWN_LANGUAGE, // ULScript_Deseret,
490 UNKNOWN_LANGUAGE, // ULScript_Inherited, [no words should be in this script]
491 UNKNOWN_LANGUAGE, // ULScript_Tagalog,
492 UNKNOWN_LANGUAGE, // ULScript_Hanunoo,
493 UNKNOWN_LANGUAGE, // ULScript_Buhid,
494 UNKNOWN_LANGUAGE, // ULScript_Tagbanwa,
495 UNKNOWN_LANGUAGE, // ULScript_Limbu,
496 UNKNOWN_LANGUAGE, // ULScript_Tai_Le,
497 UNKNOWN_LANGUAGE, // ULScript_Linear_B,
498 UNKNOWN_LANGUAGE, // ULScript_Ugaritic,
499 UNKNOWN_LANGUAGE, // ULScript_Shavian,
500 UNKNOWN_LANGUAGE, // ULScript_Osmanya,
501 UNKNOWN_LANGUAGE, // ULScript_Cypriot,
502 UNKNOWN_LANGUAGE, // ULScript_Buginese,
503 UNKNOWN_LANGUAGE, // ULScript_Coptic,
504 UNKNOWN_LANGUAGE, // ULScript_New_Tai_Lue,
505 UNKNOWN_LANGUAGE, // ULScript_Glagolitic,
506 UNKNOWN_LANGUAGE, // ULScript_Tifinagh,
507 UNKNOWN_LANGUAGE, // ULScript_Syloti_Nagri,
508 UNKNOWN_LANGUAGE, // ULScript_Old_Persian,
509 UNKNOWN_LANGUAGE, // ULScript_Kharoshthi,
510 UNKNOWN_LANGUAGE, // ULScript_Balinese,
511 UNKNOWN_LANGUAGE, // ULScript_Cuneiform,
512 UNKNOWN_LANGUAGE, // ULScript_Phoenician,
513 UNKNOWN_LANGUAGE, // ULScript_Phags_Pa,
514 UNKNOWN_LANGUAGE, // ULScript_Nko,
515
516 // Unicode 5.1
517 UNKNOWN_LANGUAGE, // ULScript_Sundanese,
518 UNKNOWN_LANGUAGE, // ULScript_Lepcha,
519 UNKNOWN_LANGUAGE, // ULScript_Ol_Chiki,
520 UNKNOWN_LANGUAGE, // ULScript_Vai,
521 UNKNOWN_LANGUAGE, // ULScript_Saurashtra,
522 UNKNOWN_LANGUAGE, // ULScript_Kayah_Li,
523 UNKNOWN_LANGUAGE, // ULScript_Rejang,
524 UNKNOWN_LANGUAGE, // ULScript_Lycian,
525 UNKNOWN_LANGUAGE, // ULScript_Carian,
526 UNKNOWN_LANGUAGE, // ULScript_Lydian,
527 UNKNOWN_LANGUAGE, // ULScript_Cham,
528 };
529
530 COMPILE_ASSERT(arraysize(kDefaultLanguagePerLScript) == ULScript_NUM_SCRIPTS,
531 kDefaultLanguagePerLScript_has_incorrect_length);
532
533
534 // True for standalone languages (only lang in a script)
535 // Subscripted by packed language number
536 // If 1, we will use nilgrams to determine language
537 static const uint8 kIsStandaloneLang[EXT_NUM_LANGUAGES + 1] = {
538 0,
539 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,0, // GREEK
540 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
541 0,1,0,0,1, 0,1,0,0,0, 0,0,1,1,0, 0,0,0,0,1, // MALAYALAM..KANNADA
542 1,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,1, // PUNJABI..SINHALESE
543 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,1,1,0, // ARMENIAN..LAOTHIAN
544
545 0,0,0,0,1, 0,1,1,1,0, 1,0,0,0,0, 0,0,0,0,0, // KHMER..ORIYA
546 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
547 0,1,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, // INUKTITUT
548
549 0,0,0,0,0, // [160..164]
550 // Add new language standalone bit just before here
551 0,0,0,0,0, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1,
552 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1,
553
554 1,1,1,1,
555 };
556
557 // True for ULScript_HanCJK
558 // (Vietnamese and Zhuang also have Latin script quadgrams)
559 // Subscripted by packed language number
560 static const uint8 kIsUnigramLang[EXT_NUM_LANGUAGES + 1] = {
561 0,
562 0,0,0,0,0, 0,0,0,1,1, 0,0,0,0,0, 0,1,0,0,0, // JAPANESE KOREAN CHINESE
563 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
564 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
565 0,0,0,0,0, 0,1,0,0,1, 0,0,0,0,0, 0,0,0,0,0, // VIETNAMESE CHINESE_T
566 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
567
568 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
569 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
570 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,0, // ZHUANG
571
572 0,0,0,0,0, // [160..164]
573 // Add new language unigram bit just before here
574
575 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
576 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, //
577
578 0,0,0,0,
579 };
580
581
582 // True for ULScript_HanCJK
583 // Subscripted by lscript number
584 static const uint8 kScoreUniPerLScript[] = {
585 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,0,0,0,0,0,0,0,
586 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
587 0,0,0,0,0,0,0,0,
588 };
589
590 COMPILE_ASSERT(arraysize(kScoreUniPerLScript) == ULScript_NUM_SCRIPTS,
591 kScoreUniPerLScript_has_incorrect_length);
592
593
594 // Defines Top40 packed languages
595
596 // Tier 0/1 Language enum list (16)
597 // ENGLISH, /*no en_GB,*/ FRENCH, ITALIAN, GERMAN, SPANISH, // E - FIGS
598 // DUTCH, CHINESE, CHINESE_T, JAPANESE, KOREAN,
599 // PORTUGUESE, RUSSIAN, POLISH, TURKISH, THAI,
600 // ARABIC,
601 //
602 // Tier 2 Language enum list (22)
603 // SWEDISH, FINNISH, DANISH, /*no pt-PT,*/ ROMANIAN, HUNGARIAN,
604 // HEBREW, INDONESIAN, CZECH, GREEK, NORWEGIAN,
605 // VIETNAMESE, BULGARIAN, CROATIAN, LITHUANIAN, SLOVAK,
606 // TAGALOG, SLOVENIAN, SERBIAN, CATALAN, LATVIAN,
607 // UKRAINIAN, HINDI,
608 //
609 // use SERBO_CROATIAN instead of BOSNIAN, SERBIAN, CROATIAN, MONTENEGRIN(21)
610 //
611 // Include IgnoreMe (TG_UNKNOWN_LANGUAGE, 25+1) as a top 40
612
613 // NOTE: packed, i.e. Language enum + 1
614 static const uint8 kIsPackedTop40[EXT_NUM_LANGUAGES + 1] = {
615 0,
616 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,0,
617 1,1,1,1,0, 1,0,1,0,0, 0,0,1,1,1, 1,0,0,1,0,
618 0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,1, 1,0,0,0,0,
619 0,0,0,1,0, 0,1,0,1,1, 0,0,0,0,0, 0,0,0,0,0,
620 0,0,0,0,0, 0,0,0,0,0, 0,0,1,0,0, 0,0,0,0,0,
621
622 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
623 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
624 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
625
626 0,0,0,0,0, // [160..164]
627 // Add new language top40 bit just before here
628
629 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
630 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
631
632 0,0,0,0,
633 };
634
635
636
637 // Table has 234 eight-byte entries. Each entry has a five-byte array and
638 // a three-byte array of log base 2 probabilities in the range 0..11.
639 // The intended use is to express five or three probabilities in a single-byte
640 // subscript, then decode via this table. These probabilities are
641 // intended to go with an array of five or three language numbers.
642 //
643 // The corresponding language numbers will have to be sorted by descending
644 // probability, then the actual probability subscript chosen to match the
645 // closest available entry in this table.
646 //
647 // Pattern of probability values:
648 // hi 3/4 1/2 1/4 lo hi mid lo
649 // where "3/4" is (hi*3+lo)/4, "1/2" is (hi+lo)/2, and "1/4" is (hi+lo*3)/4 and
650 // mid is one of 3/4 1/2 or 1/4.
651 // There are three groups of 78 (=12*13/2) entries, with hi running 0..11 and
652 // lo running 0..hi. Only the first group is used for five-entry lookups.
653 // The mid value in the first group is 1/2, the second group 3/4, and the
654 // third group 1/4. For three-entry lookups, this allows the mid entry to be
655 // somewhat higher or lower than the midpoint, to allow a better match to the
656 // original probabilities.
657 static const int kLgProbV2TblSize = 234;
658 static const uint8 kLgProbV2Tbl[kLgProbV2TblSize * 8] = {
659 1,1,1,1,1, 1,1,1, // [0]
660 2,2,2,1,1, 2,2,1, // [1]
661 2,2,2,2,2, 2,2,2,
662 3,3,2,2,1, 3,2,1, // [3]
663 3,3,3,2,2, 3,3,2,
664 3,3,3,3,3, 3,3,3,
665 4,3,3,2,1, 4,3,1, // [6]
666 4,4,3,3,2, 4,3,2,
667 4,4,4,3,3, 4,4,3,
668 4,4,4,4,4, 4,4,4,
669 5,4,3,2,1, 5,3,1, // [10]
670 5,4,4,3,2, 5,4,2,
671 5,5,4,4,3, 5,4,3,
672 5,5,5,4,4, 5,5,4,
673 5,5,5,5,5, 5,5,5,
674 6,5,4,2,1, 6,4,1, // [15]
675 6,5,4,3,2, 6,4,2,
676 6,5,5,4,3, 6,5,3,
677 6,6,5,5,4, 6,5,4,
678 6,6,6,5,5, 6,6,5,
679 6,6,6,6,6, 6,6,6,
680 7,6,4,3,1, 7,4,1, // [21]
681 7,6,5,3,2, 7,5,2,
682 7,6,5,4,3, 7,5,3,
683 7,6,6,5,4, 7,6,4,
684 7,7,6,6,5, 7,6,5,
685 7,7,7,6,6, 7,7,6,
686 7,7,7,7,7, 7,7,7,
687 8,6,5,3,1, 8,5,1, // [28]
688 8,7,5,4,2, 8,5,2,
689 8,7,6,4,3, 8,6,3,
690 8,7,6,5,4, 8,6,4,
691 8,7,7,6,5, 8,7,5,
692 8,8,7,7,6, 8,7,6,
693 8,8,8,7,7, 8,8,7,
694 8,8,8,8,8, 8,8,8,
695 9,7,5,3,1, 9,5,1, // [36]
696 9,7,6,4,2, 9,6,2,
697 9,8,6,5,3, 9,6,3,
698 9,8,7,5,4, 9,7,4,
699 9,8,7,6,5, 9,7,5,
700 9,8,8,7,6, 9,8,6,
701 9,9,8,8,7, 9,8,7,
702 9,9,9,8,8, 9,9,8,
703 9,9,9,9,9, 9,9,9,
704 10,8,6,3,1, 10,6,1, // [45]
705 10,8,6,4,2, 10,6,2,
706 10,8,7,5,3, 10,7,3,
707 10,9,7,6,4, 10,7,4,
708 10,9,8,6,5, 10,8,5,
709 10,9,8,7,6, 10,8,6,
710 10,9,9,8,7, 10,9,7,
711 10,10,9,9,8, 10,9,8,
712 10,10,10,9,9, 10,10,9,
713 10,10,10,10,10, 10,10,10,
714 11,9,6,4,1, 11,6,1, // [55]
715 11,9,7,4,2, 11,7,2,
716 11,9,7,5,3, 11,7,3,
717 11,9,8,6,4, 11,8,4,
718 11,10,8,7,5, 11,8,5,
719 11,10,9,7,6, 11,9,6,
720 11,10,9,8,7, 11,9,7,
721 11,10,10,9,8, 11,10,8,
722 11,11,10,10,9, 11,10,9,
723 11,11,11,10,10, 11,11,10,
724 11,11,11,11,11, 11,11,11,
725 12,9,7,4,1, 12,7,1, // [66]
726 12,10,7,5,2, 12,7,2,
727 12,10,8,5,3, 12,8,3,
728 12,10,8,6,4, 12,8,4,
729 12,10,9,7,5, 12,9,5,
730 12,11,9,8,6, 12,9,6,
731 12,11,10,8,7, 12,10,7,
732 12,11,10,9,8, 12,10,8,
733 12,11,11,10,9, 12,11,9,
734 12,12,11,11,10, 12,11,10,
735 12,12,12,11,11, 12,12,11,
736 12,12,12,12,12, 12,12,12,
737
738 1,1,1,1,1, 1,1,1,
739 2,2,2,1,1, 2,2,1,
740 2,2,2,2,2, 2,2,2,
741 3,3,2,2,1, 3,3,1,
742 3,3,3,2,2, 3,3,2,
743 3,3,3,3,3, 3,3,3,
744 4,3,3,2,1, 4,3,1,
745 4,4,3,3,2, 4,4,2,
746 4,4,4,3,3, 4,4,3,
747 4,4,4,4,4, 4,4,4,
748 5,4,3,2,1, 5,4,1,
749 5,4,4,3,2, 5,4,2,
750 5,5,4,4,3, 5,5,3,
751 5,5,5,4,4, 5,5,4,
752 5,5,5,5,5, 5,5,5,
753 6,5,4,2,1, 6,5,1,
754 6,5,4,3,2, 6,5,2,
755 6,5,5,4,3, 6,5,3,
756 6,6,5,5,4, 6,6,4,
757 6,6,6,5,5, 6,6,5,
758 6,6,6,6,6, 6,6,6,
759 7,6,4,3,1, 7,6,1,
760 7,6,5,3,2, 7,6,2,
761 7,6,5,4,3, 7,6,3,
762 7,6,6,5,4, 7,6,4,
763 7,7,6,6,5, 7,7,5,
764 7,7,7,6,6, 7,7,6,
765 7,7,7,7,7, 7,7,7,
766 8,6,5,3,1, 8,6,1,
767 8,7,5,4,2, 8,7,2,
768 8,7,6,4,3, 8,7,3,
769 8,7,6,5,4, 8,7,4,
770 8,7,7,6,5, 8,7,5,
771 8,8,7,7,6, 8,8,6,
772 8,8,8,7,7, 8,8,7,
773 8,8,8,8,8, 8,8,8,
774 9,7,5,3,1, 9,7,1,
775 9,7,6,4,2, 9,7,2,
776 9,8,6,5,3, 9,8,3,
777 9,8,7,5,4, 9,8,4,
778 9,8,7,6,5, 9,8,5,
779 9,8,8,7,6, 9,8,6,
780 9,9,8,8,7, 9,9,7,
781 9,9,9,8,8, 9,9,8,
782 9,9,9,9,9, 9,9,9,
783 10,8,6,3,1, 10,8,1,
784 10,8,6,4,2, 10,8,2,
785 10,8,7,5,3, 10,8,3,
786 10,9,7,6,4, 10,9,4,
787 10,9,8,6,5, 10,9,5,
788 10,9,8,7,6, 10,9,6,
789 10,9,9,8,7, 10,9,7,
790 10,10,9,9,8, 10,10,8,
791 10,10,10,9,9, 10,10,9,
792 10,10,10,10,10, 10,10,10,
793 11,9,6,4,1, 11,9,1,
794 11,9,7,4,2, 11,9,2,
795 11,9,7,5,3, 11,9,3,
796 11,9,8,6,4, 11,9,4,
797 11,10,8,7,5, 11,10,5,
798 11,10,9,7,6, 11,10,6,
799 11,10,9,8,7, 11,10,7,
800 11,10,10,9,8, 11,10,8,
801 11,11,10,10,9, 11,11,9,
802 11,11,11,10,10, 11,11,10,
803 11,11,11,11,11, 11,11,11,
804 12,9,7,4,1, 12,9,1,
805 12,10,7,5,2, 12,10,2,
806 12,10,8,5,3, 12,10,3,
807 12,10,8,6,4, 12,10,4,
808 12,10,9,7,5, 12,10,5,
809 12,11,9,8,6, 12,11,6,
810 12,11,10,8,7, 12,11,7,
811 12,11,10,9,8, 12,11,8,
812 12,11,11,10,9, 12,11,9,
813 12,12,11,11,10, 12,12,10,
814 12,12,12,11,11, 12,12,11,
815 12,12,12,12,12, 12,12,12,
816
817 1,1,1,1,1, 1,1,1,
818 2,2,2,1,1, 2,1,1,
819 2,2,2,2,2, 2,2,2,
820 3,3,2,2,1, 3,2,1,
821 3,3,3,2,2, 3,2,2,
822 3,3,3,3,3, 3,3,3,
823 4,3,3,2,1, 4,2,1,
824 4,4,3,3,2, 4,3,2,
825 4,4,4,3,3, 4,3,3,
826 4,4,4,4,4, 4,4,4,
827 5,4,3,2,1, 5,2,1,
828 5,4,4,3,2, 5,3,2,
829 5,5,4,4,3, 5,4,3,
830 5,5,5,4,4, 5,4,4,
831 5,5,5,5,5, 5,5,5,
832 6,5,4,2,1, 6,2,1,
833 6,5,4,3,2, 6,3,2,
834 6,5,5,4,3, 6,4,3,
835 6,6,5,5,4, 6,5,4,
836 6,6,6,5,5, 6,5,5,
837 6,6,6,6,6, 6,6,6,
838 7,6,4,3,1, 7,3,1,
839 7,6,5,3,2, 7,3,2,
840 7,6,5,4,3, 7,4,3,
841 7,6,6,5,4, 7,5,4,
842 7,7,6,6,5, 7,6,5,
843 7,7,7,6,6, 7,6,6,
844 7,7,7,7,7, 7,7,7,
845 8,6,5,3,1, 8,3,1,
846 8,7,5,4,2, 8,4,2,
847 8,7,6,4,3, 8,4,3,
848 8,7,6,5,4, 8,5,4,
849 8,7,7,6,5, 8,6,5,
850 8,8,7,7,6, 8,7,6,
851 8,8,8,7,7, 8,7,7,
852 8,8,8,8,8, 8,8,8,
853 9,7,5,3,1, 9,3,1,
854 9,7,6,4,2, 9,4,2,
855 9,8,6,5,3, 9,5,3,
856 9,8,7,5,4, 9,5,4,
857 9,8,7,6,5, 9,6,5,
858 9,8,8,7,6, 9,7,6,
859 9,9,8,8,7, 9,8,7,
860 9,9,9,8,8, 9,8,8,
861 9,9,9,9,9, 9,9,9,
862 10,8,6,3,1, 10,3,1,
863 10,8,6,4,2, 10,4,2,
864 10,8,7,5,3, 10,5,3,
865 10,9,7,6,4, 10,6,4,
866 10,9,8,6,5, 10,6,5,
867 10,9,8,7,6, 10,7,6,
868 10,9,9,8,7, 10,8,7,
869 10,10,9,9,8, 10,9,8,
870 10,10,10,9,9, 10,9,9,
871 10,10,10,10,10, 10,10,10,
872 11,9,6,4,1, 11,4,1,
873 11,9,7,4,2, 11,4,2,
874 11,9,7,5,3, 11,5,3,
875 11,9,8,6,4, 11,6,4,
876 11,10,8,7,5, 11,7,5,
877 11,10,9,7,6, 11,7,6,
878 11,10,9,8,7, 11,8,7,
879 11,10,10,9,8, 11,9,8,
880 11,11,10,10,9, 11,10,9,
881 11,11,11,10,10, 11,10,10,
882 11,11,11,11,11, 11,11,11,
883 12,9,7,4,1, 12,4,1,
884 12,10,7,5,2, 12,5,2,
885 12,10,8,5,3, 12,5,3,
886 12,10,8,6,4, 12,6,4,
887 12,10,9,7,5, 12,7,5,
888 12,11,9,8,6, 12,8,6,
889 12,11,10,8,7, 12,8,7,
890 12,11,10,9,8, 12,9,8,
891 12,11,11,10,9, 12,10,9,
892 12,12,11,11,10, 12,11,10,
893 12,12,12,11,11, 12,11,11,
894 12,12,12,12,12, 12,12,12,
895 };
896
897 // Backmap a single desired probability into an entry in kLgProbV2Tbl
898 static const uint8 kLgProbV2TblBackmap[13] = {
899 0,
900 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66,
901 };
902
903
904 // Always advances one UTF-8 character
905 static const uint8 kAdvanceOneChar[256] = {
906 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
907 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
908 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
909 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
910
911 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
912 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
913 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
914 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
915 };
916
917 // Does not advance past space or cr/lf/nul
918 static const uint8 kAdvanceOneCharButSpace[256] = {
919 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
920 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
921 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
922 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
923
924 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
925 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
926 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
927 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
928 };
929
930 // Advances *only* on space or ASCII vowel (or illegal byte)
931 static const uint8 kAdvanceOneCharSpaceVowel[256] = {
932 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
933 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
934 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
935 0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
936
937 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
938 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
939 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
940 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
941 };
942
943 // Advances *only* on space (or illegal byte)
944 static const uint8 kAdvanceOneCharSpace[256] = {
945 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
946 1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
947 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
948 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
949
950 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
951 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
952 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
953 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
954 };
955
956
957 //------------------------------------------------------------------------------
958 // General
959 //------------------------------------------------------------------------------
960 static inline int minint(int a, int b) {return (a < b) ? a: b;}
961 static inline int maxint(int a, int b) {return (a > b) ? a: b;}
962
963 // Here to make available for debugging
964 int ReliabilityDelta(int value1, int value2, int count);
965 int ReliabilityMainstream(int topscore, int len, int mean_score);
966
967 // Returns "0" for too small
968 inline const char* MyExtLanguageCode(Language lang) {
969 return ExtLanguageCode(lang);
970 }
971
972 // Map script into Latin, Cyrillic, Arabic, Other. Used in keeping track of
973 // amount of training data for language-script combinations
974 inline int LScript4(UnicodeLScript lscript) {
975 if (lscript == ULScript_Latin) {return 0;}
976 if (lscript == ULScript_Cyrillic) {return 1;}
977 if (lscript == ULScript_Arabic) {return 2;}
978 return 3;
979 }
980
981
982 // Routines to access 3 or 5 log probabilities in a single byte.
983
984 // Return address of 8-byte entry[i]
985 inline const uint8* LgProb2TblEntry(int i) {
986 return &kLgProbV2Tbl[i * 8];
987 }
988
989 // Return one of five probabilities in an entry
990 // CURRENTLY UNUSED
991 inline uint8 LgProb5(const uint8* entry, int j) {
992 return entry[j];
993 }
994
995 // Return one of three probabilities in an entry
996 inline uint8 LgProb3(const uint8* entry, int j) {
997 return entry[j + 5];
998 }
999
1000
1001
1002 //------------------------------------------------------------------------------
1003 // Hashing groups of 1/2/4/8 letters, perhaps with spaces or underscores
1004 //------------------------------------------------------------------------------
1005
1006 // Pick up 1..12 bytes and hash them via mask/shift/add. NO pre/post
1007 // OVERSHOOTS up to 3 bytes
1008 uint32 BiHashV25(const char* word_ptr, int bytecount);
1009
1010 // Pick up 1..12 bytes plus pre/post space and hash them via mask/shift/add
1011 // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
1012 uint32 QuadHashV25(const char* word_ptr, int bytecount);
1013
1014 // Pick up 1..12 bytes plus pre/post '_' and hash them via mask/shift/add
1015 // OVERSHOOTS up to 3 bytes
1016 uint32 QuadHashV25Underscore(const char* word_ptr, int bytecount);
1017
1018
1019 // Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add
1020 // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
1021 // For runtime use of tables V3
1022 uint64 OctaHash40(const char* word_ptr, int bytecount);
1023
1024 uint64 OctaHash40underscore(const char* word_ptr, int bytecount);
1025
1026
1027 // From 32-bit gram FP, return hash table subscript and remaining key
1028 inline void QuadFPJustHash(uint32 quadhash,
1029 uint32 keymask,
1030 int bucketcount,
1031 uint32* subscr, uint32* hashkey) {
1032 *subscr = (quadhash + (quadhash >> 12)) & (bucketcount - 1);
1033 *hashkey = quadhash & keymask;
1034 }
1035
1036 // Look up 32-bit gram FP in caller-passed table
1037 // Typical size 256K entries (1.5MB)
1038 // Two-byte hashkey
1039 inline const uint32 QuadHashV3Lookup4(const cld::CLDTableSummary* gram_obj,
1040 uint32 quadhash) {
1041
1042 uint32 subscr, hashkey;
1043 const IndirectProbBucket4* quadtable = gram_obj->kCLDTable;
1044 uint32 keymask = gram_obj->kCLDTableKeyMask;
1045 int bucketcount = gram_obj->kCLDTableSize;
1046 QuadFPJustHash(quadhash, keymask, bucketcount, &subscr, &hashkey);
1047 const IndirectProbBucket4* bucket_ptr = &quadtable[subscr];
1048 // Four-way associative, 4 compares
1049 if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) {
1050 return bucket_ptr->keyvalue[0];
1051 }
1052 if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) {
1053 return bucket_ptr->keyvalue[1];
1054 }
1055 if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) {
1056 return bucket_ptr->keyvalue[2];
1057 }
1058 if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) {
1059 return bucket_ptr->keyvalue[3];
1060 }
1061 return 0;
1062 }
1063
1064
1065 // Map 40 bits to subscript, hashkey, expected 18-22 bit subscript (min 16)
1066 // wwwwwwww xxxxxxxx xxxxxxxx yyyyyyyy yyyyyyyy
1067 // + ........ ....wwww wwwwxxxx xxxxxxxx xxxxyyyy
1068 // 00000000 00000000 00000011 11111111 11111111 (18-bit bucketcount-1)
1069 //
1070 // hashkey:
1071 // wwwwxxxx xxxxxxxx xxxx.... ........ (20-bit keymask)
1072 // 12-bit shift in subscript mixes in ~4 letters x 4 bits each
1073
1074 // From 40-bit gram FP, return hash table subscript and remaining key
1075 inline void OctaFPJustHash(uint64 longwordhash,
1076 uint32 keymask,
1077 int bucketcount,
1078 uint32* subscr, uint32* hashkey) {
1079 uint32 temp = (longwordhash + (longwordhash >> 12)) & (bucketcount - 1);
1080 *subscr = temp;
1081 temp = longwordhash >> 4;
1082 *hashkey = temp & keymask;
1083 }
1084
1085 // Look up 40-bit gram FP in caller-passed table
1086 // Typical size 256K-4M entries (1-16MB)
1087 // 24-12 bit hashkey packed with 8-20 bit indirect lang/probs
1088 // keymask is 0xfffff000 for 20-bit hashkey and 12-bit indirect
1089 inline const uint32 OctaHashV3Lookup4(const cld::CLDTableSummary* gram_obj,
1090 uint64 longwordhash) {
1091 uint32 subscr, hashkey;
1092 const IndirectProbBucket4* octatable = gram_obj->kCLDTable;
1093 uint32 keymask = gram_obj->kCLDTableKeyMask;
1094 int bucketcount = gram_obj->kCLDTableSize;
1095 OctaFPJustHash(longwordhash, keymask, bucketcount,
1096 &subscr, &hashkey);
1097 const IndirectProbBucket4* bucket_ptr = &octatable[subscr];
1098 // Four-way associative, 4 compares
1099 if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) {
1100 return bucket_ptr->keyvalue[0];
1101 }
1102 if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) {
1103 return bucket_ptr->keyvalue[1];
1104 }
1105 if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) {
1106 return bucket_ptr->keyvalue[2];
1107 }
1108 if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) {
1109 return bucket_ptr->keyvalue[3];
1110 }
1111 return 0;
1112 }
1113
1114
1115
1116 //------------------------------------------------------------------------------
1117 // Scoring single groups of letters
1118 //------------------------------------------------------------------------------
1119
1120 // UNIGRAM score one => tote
1121 // Input: 1-byte entry of subscript into unigram probs, plus
1122 // an accumulator tote.
1123 // Output: running sums in tote updated
1124 void ProcessProbV25UniTote(int propval, Tote* tote);
1125
1126 // BIGRAM, QUADGRAM, OCTAGRAM score one => tote
1127 // Input: 4-byte entry of 3 language numbers and one probability subscript,
1128 // plus an accumulator tote. (language 0 means unused entry)
1129 // Output: running sums in tote updated
1130 void ProcessProbV25Tote(uint32 probs, Tote* tote);
1131
1132
1133 //------------------------------------------------------------------------------
1134 // Routines to accumulate probabilities
1135 //------------------------------------------------------------------------------
1136
1137 // Score up to n=gram_limit unigrams, returning number of bytes consumed
1138 // Caller supplies table, such as compact_lang_det_generated_ctjkvz_b1_obj
1139 int DoUniScoreV3(const UTF8PropObj* unigram_obj,
1140 const char* isrc, int srclen, int advance_by,
1141 int* tote_grams, int gram_limit, Tote* chunk_tote);
1142
1143
1144 // Score all words in isrc, using languages that have bigrams (CJK)
1145 // Caller supplies table, such as &kCjkBiTable_obj or &kGibberishTable_obj
1146 // Return number of bigrams that hit in the hash table
1147 int DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj,
1148 const char* isrc, int srclen, Tote* chunk_tote);
1149
1150
1151 // Score up to n=gram_limit quadgrams, returning number of bytes consumed
1152 // Caller supplies table, such as &kQuadTable_obj or &kGibberishTable_obj
1153 int DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj,
1154 const char* isrc, int srclen, int advance_by,
1155 int* tote_grams, int gram_limit, Tote* chunk_tote);
1156
1157 // Score all octagrams (words) in isrc, using languages that have quadgrams
1158 // Caller supplies table, such as &kLongWord8Table_obj
1159 // Return number of words that hit in the hash table
1160 int DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj,
1161 const char* isrc, int srclen, Tote* chunk_tote);
1162
1163 //------------------------------------------------------------------------------
1164 // Reliability calculations, for single language and between languages
1165 //------------------------------------------------------------------------------
1166
1167 // Reliability = 0..100
1168 static const int kMinReliable = 75;
1169
1170 // Calculate ratio of score per 1KB vs. expected score per 1KB
1171 double GetNormalizedScore(Language lang, UnicodeLScript lscript,
1172 int bytes, int score);
1173
1174 // Calculate reliablity of len bytes of script lscript with chunk_tote
1175 int GetReliability(int len, UnicodeLScript lscript, const Tote* chunk_tote);
1176
1177
1178 //------------------------------------------------------------------------------
1179 // Miscellaneous
1180 //------------------------------------------------------------------------------
1181
1182 // Make languages packed into uint32 values non-zero
1183 // These routines later could remap so languages not in QuadHash tables are not
1184 // represented, and so that any thrashing in accumulation is eliminated
1185 uint8 inline PackLanguage(Language lang) {
1186 return static_cast<uint8>(lang + 1);}
1187
1188 Language inline UnpackLanguage(int ilang) {
1189 return static_cast<Language>(ilang - 1);}
1190
1191 // Useful single-byte tests
1192 bool inline IsUTF8ContinueByte(char c) {
1193 return static_cast<signed char>(c) < -64;}
1194 bool inline IsUTF8HighByte(char c) {
1195 return static_cast<signed char>(c) < 0;}
1196
1197
1198 // Demote all languages except Top40 and plus_one
1199 // Do this just before sorting
1200 void DemoteNotTop40(Tote* chunk_tote, int packed_plus_one);
1201
1202 } // End namespace cld
1203
1204
1205 #endif // ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_