?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch '138-toggle-free-look-with-hotkey' into 'main/atys-live'
[ryzomcore.git] / nel / src / misc / i18n.cpp
blob188d672b3106c03f2818e702d82d7835628ea8e6
1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010-2019 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2012 Laszlo KIS-ADAM (dfighter) <dfighter1985@gmail.com>
6 // Copyright (C) 2014 Matthew LAGOE (Botanic) <cyberempires@gmail.com>
7 // Copyright (C) 2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
8 //
9 // This program is free software: you can redistribute it and/or modify
10 // it under the terms of the GNU Affero General Public License as
11 // published by the Free Software Foundation, either version 3 of the
12 // License, or (at your option) any later version.
13 //
14 // This program is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 // GNU Affero General Public License for more details.
18 //
19 // You should have received a copy of the GNU Affero General Public License
20 // along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22
23 #include "stdmisc.h"
24
25 #include "nel/misc/path.h"
26 #include "nel/misc/i18n.h"
27
28 #include <locale.h>
29
30 #ifdef NL_OS_MAC
31 #include <CoreFoundation/CoreFoundation.h>
32 #endif
33
34 using namespace std;
35
36 #ifdef DEBUG_NEW
37 #define new DEBUG_NEW
38 #endif
39
40 namespace NLMISC {
41
42 CI18N::StrMapContainer CI18N::_StrMap;
43 CI18N::StrMapContainer CI18N::_StrMapFallback;
44 CI18N::StrMapContainer16 CI18N::_StrMap16;
45 CI18N::StrMapContainer16 CI18N::_StrMapFallback16;
46 bool CI18N::_StrMapLoaded = false;
47 const ucstring CI18N::_NotTranslatedValue16("<Not Translated>");
48 const std::string CI18N::_NotTranslatedValue("<Not Translated>");
49 bool CI18N::_LanguagesNamesLoaded = false;
50 string CI18N::_SelectedLanguageCode;
51 CI18N::ILoadProxy *CI18N::_LoadProxy = 0;
52 vector<string> CI18N::_LanguageCodes;
53 vector<std::string> CI18N::_LanguageNames;
54 std::string CI18N::_SystemLanguageCode;
55 bool CI18N::noResolution = false;
56
57 void CI18N::setLoadProxy(ILoadProxy *loadProxy)
58 {
59 _LoadProxy = loadProxy;
60 }
61
62 void CI18N::initLanguages()
63 {
64 if (!_LanguagesNamesLoaded)
65 {
66 _LanguageCodes.push_back("en");
67 _LanguageCodes.push_back("fr");
68 _LanguageCodes.push_back("de");
69 _LanguageCodes.push_back("ru");
70 _LanguageCodes.push_back("es");
71
72 _LanguageNames.push_back("English");
73 _LanguageNames.push_back("French");
74 _LanguageNames.push_back("German");
75 _LanguageNames.push_back("Russian");
76 _LanguageNames.push_back("Spanish");
77
78 _LanguagesNamesLoaded = true;
79 }
80 }
81
82 const std::vector<std::string> &CI18N::getLanguageNames()
83 {
84 initLanguages();
85
86 return _LanguageNames;
87 }
88
89 const std::vector<std::string> &CI18N::getLanguageCodes()
90 {
91 initLanguages();
92
93 return _LanguageCodes;
94 }
95
96 void CI18N::load (const string &languageCode, const string &fallbackLanguageCode)
97 {
98 if (_StrMapLoaded)
99 {
100 _StrMap.clear();
101 _StrMap16.clear();
102 }
103 else
104 {
105 _StrMapLoaded = true;
106 }
107 _SelectedLanguageCode = languageCode;
108 loadFileIntoMap(languageCode + ".uxt", _StrMap, _StrMap16);
109
110 _StrMapFallback.clear();
111 _StrMapFallback16.clear();
112 if(!fallbackLanguageCode.empty())
113 {
114 loadFileIntoMap(fallbackLanguageCode + ".uxt", _StrMapFallback, _StrMapFallback16);
115 }
116 }
117
118 bool CI18N::loadFileIntoMap(const string &fileName, StrMapContainer &destMap, StrMapContainer16 &destMap16)
119 {
120 ucstring text;
121 // read in the text
122 if (_LoadProxy)
123 _LoadProxy->loadStringFile(fileName, text);
124 else
125 readTextFile(fileName, text);
126
127 // remove any comment
128 removeCComment(text);
129
130 ucstring::const_iterator first(text.begin()), last(text.end());
131 string lastReadLabel("nothing");
132
133 while (first != last)
134 {
135 skipWhiteSpace(first, last);
136 string label;
137 ucstring ucs;
138 if (!parseLabel(first, last, label))
139 {
140 nlwarning("I18N: Error reading label field in %s. Stop reading after %s.", fileName.c_str(), lastReadLabel.c_str());
141 return false;
142 }
143 lastReadLabel = label;
144 skipWhiteSpace(first, last);
145 if (!parseMarkedString('[', ']', first, last, ucs))
146 {
147 nlwarning("I18N: Error reading text for label %s in %s. Stop reading.", label.c_str(), fileName.c_str());
148 return false;
149 }
150
151 // ok, a line read.
152 pair<map<string, ucstring>::iterator, bool> ret;
153 ret = destMap16.insert(make_pair(label, ucs));
154 if (!ret.second)
155 {
156 nlwarning("I18N: Error in %s, the label %s exists twice !", fileName.c_str(), label.c_str());
157 }
158 destMap.insert(make_pair(label, ucs.toUtf8()));
159 skipWhiteSpace(first, last);
160 }
161
162 // a little check to ensure that the lang name has been set.
163 StrMapContainer::iterator it(destMap.find("LanguageName"));
164 if (it == destMap.end())
165 {
166 nlwarning("I18N: In file %s, missing LanguageName translation (should be first in file)", fileName.c_str());
167 }
168 nlassert(destMap.size() == destMap16.size());
169 return true;
170 }
171
172 void CI18N::loadFromFilename(const string &filename, bool reload)
173 {
174 StrMapContainer destMap;
175 StrMapContainer16 destMap16;
176 if (!loadFileIntoMap(filename, destMap, destMap16))
177 {
178 return;
179 }
180 // merge with existing map
181 for(StrMapContainer::iterator it = destMap.begin(); it != destMap.end(); ++it)
182 {
183 if (!reload)
184 {
185 if (_StrMap16.count(it->first))
186 {
187 nlwarning("I18N: Error in %s, the label %s exist twice !", filename.c_str(), it->first.c_str());
188 }
189 }
190 _StrMap16[it->first] = ucstring::makeFromUtf8(it->second);
191 _StrMap[it->first] = it->second;
192 }
193 }
194
195 const std::string &CI18N::get(const string &label)
196 {
197 if (noResolution)
198 {
199 return label;
200 }
201
202 if (label.empty())
203 {
204 static const std::string empty;
205 return empty;
206 }
207
208 StrMapContainer::iterator it(_StrMap.find(label));
209
210 if (it != _StrMap.end())
211 return it->second;
212
213 static CHashSet<string> missingStrings;
214 if (missingStrings.find(label) == missingStrings.end())
215 {
216 nlwarning("I18N: The string %s did not exist in language %s (display once)", label.c_str(), _SelectedLanguageCode.c_str());
217 missingStrings.insert(label);
218 }
219
220 // use the fall back language if it exists
221 it = _StrMapFallback.find(label);
222 if (it != _StrMapFallback.end())
223 return it->second;
224
225 static std::string badString;
226 badString = string("<NotExist:") + label + ">";
227 return badString;
228 }
229
230 const ucstring &CI18N::getAsUtf16 (const string &label)
231 {
232 if( noResolution )
233 {
234 static ucstring labelString;
235 labelString = label;
236 return labelString;
237 }
238
239 if (label.empty())
240 {
241 static const ucstring emptyString;
242 return emptyString;
243 }
244
245 StrMapContainer16::iterator it(_StrMap16.find(label));
246
247 if (it != _StrMap16.end())
248 return it->second;
249
250 static CHashSet<string> missingStrings;
251 if (missingStrings.find(label) == missingStrings.end())
252 {
253 nlwarning("I18N: The string %s did not exist in language %s (display once)", label.c_str(), _SelectedLanguageCode.c_str());
254 missingStrings.insert(label);
255 }
256
257 // use the fall back language if it exists
258 it = _StrMapFallback16.find(label);
259 if (it != _StrMapFallback16.end())
260 return it->second;
261
262 static ucstring badString;
263
264 badString = ucstring(string("<NotExist:")+label+">");
265
266 return badString;
267 }
268
269 bool CI18N::hasTranslation(const string &label)
270 {
271 if (label.empty()) return true;
272
273 if(_StrMap.find(label) != _StrMap.end())
274 return true;
275
276 // use the fall back language if it exists
277 if (_StrMapFallback.find(label) != _StrMapFallback.end())
278 return true;
279
280 return false;
281 }
282
283 std::string CI18N::getCurrentLanguageName ()
284 {
285 return get("LanguageName");
286 }
287
288 string CI18N::getCurrentLanguageCode ()
289 {
290 return _SelectedLanguageCode;
291 }
292
293 bool CI18N::isLanguageCodeSupported(const std::string &lang)
294 {
295 initLanguages();
296
297 for (sint i = 0, ilen = _LanguageCodes.size(); i < ilen; ++i)
298 {
299 if (lang == _LanguageCodes[i]) return true;
300 }
301
302 return false;
303 }
304
305 std::string CI18N::getSystemLanguageCode ()
306 {
307 if (!_SystemLanguageCode.empty())
308 return _SystemLanguageCode;
309
310 #ifdef NL_OS_MAC
311 // under OS X, locale is only defined in console, not in UI
312 // so we need to use CoreFoundation API to retrieve it
313
314 // get an array with all preferred languages
315 CFArrayRef langs = CFLocaleCopyPreferredLanguages();
316
317 if (langs)
318 {
319 // get languages count
320 sint languagesCount = CFArrayGetCount(langs);
321
322 // process each language
323 for (sint i = 0; i < languagesCount; ++i)
324 {
325 std::string lang;
326
327 // get language CFString
328 CFStringRef langCF = (CFStringRef)CFArrayGetValueAtIndex(langs, i);
329
330 if (langCF)
331 {
332 // get a C string from CFString
333 const char *langStr = CFStringGetCStringPtr(langCF, kCFStringEncodingASCII);
334
335 if (!langStr)
336 {
337 // get length of the CFString
338 CFIndex length = CFStringGetLength(langCF);
339
340 // allocate a temporary buffer to hold converted string
341 char *tmp = new char[length+1];
342
343 // use alternative function to get a C string from CFString
344 if (CFStringGetCString(langCF, tmp, length+1, kCFStringEncodingASCII))
345 {
346 lang = std::string(tmp, length);
347 }
348 else
349 {
350 nlwarning("Unable to convert CFStringRef to string");
351 }
352
353 delete [] tmp;
354 }
355 else
356 {
357 lang = std::string(langStr);
358 }
359
360 CFRelease(langCF);
361 }
362
363 if (!lang.empty())
364 {
365 // fix language code if country is specified
366 std::string::size_type pos = lang.find('-');
367
368 if (pos != std::string::npos)
369 lang = lang.substr(0, pos);
370
371 // only keep language code if supported by NeL
372 if (isLanguageCodeSupported(lang))
373 {
374 _SystemLanguageCode = lang;
375 break;
376 }
377 }
378 }
379
380 // don't need languages array anymore
381 CFRelease(langs);
382 }
383 #endif
384
385 #ifdef NL_OS_WINDOWS
386 // use user locale under Windows (since Vista)
387 if (_SystemLanguageCode.empty())
388 {
389 // GetUserDefaultLocaleName prototype
390 typedef int (WINAPI* GetUserDefaultLocaleNamePtr)(LPWSTR lpLocaleName, int cchLocaleName);
391
392 // get pointer on GetUserDefaultLocaleName, kernel32.dll is always in memory so no need to call LoadLibrary
393 HMODULE hKernel32 = GetModuleHandleA("kernel32.dll");
394 if (hKernel32)
395 {
396 GetUserDefaultLocaleNamePtr nlGetUserDefaultLocaleName = (GetUserDefaultLocaleNamePtr)GetProcAddress(hKernel32, "GetUserDefaultLocaleName");
397
398 // only use it if found
399 if (nlGetUserDefaultLocaleName)
400 {
401 // get user locale
402 wchar_t buffer[LOCALE_NAME_MAX_LENGTH];
403 sint res = nlGetUserDefaultLocaleName(buffer, LOCALE_NAME_MAX_LENGTH);
404
405 // convert wide string to std::string
406 std::string lang = wideToUtf8(buffer);
407
408 // only keep 2 first characters
409 if (lang.size() > 1)
410 _SystemLanguageCode = lang.substr(0, 2);
411 }
412 }
413 }
414 #endif
415
416 // use system locale (works under OS X, Linux and Windows)
417 if (_SystemLanguageCode.empty())
418 {
419 // get default locale
420 char *locale = setlocale(LC_CTYPE, "");
421
422 if (locale)
423 {
424 std::string lang(locale);
425
426 #ifdef NL_OS_WINDOWS
427 // be sure supported languages are initialized
428 initLanguages();
429
430 // locales names are different under Windows, for example: French_France.1252
431 for(uint i = 0; i < _LanguageNames.size(); ++i)
432 {
433 std::string name = _LanguageNames[i];
434
435 // so we compare the language name with the supported ones
436 if (lang.compare(0, name.length(), name) == 0)
437 {
438 // found, so use its code
439 _SystemLanguageCode = _LanguageCodes[i];
440 break;
441 }
442 }
443 #else
444 if (lang.size() > 1)
445 {
446 // only keep 2 first characters
447 lang = NLMISC::toLowerAscii(lang).substr(0, 2);
448
449 // language code supported?
450 if (isLanguageCodeSupported(lang))
451 _SystemLanguageCode = lang;
452 }
453 #endif
454 }
455 }
456
457 // english is default language
458 if (_SystemLanguageCode.empty())
459 _SystemLanguageCode = "en";
460
461 return _SystemLanguageCode;
462 }
463
464 bool CI18N::setSystemLanguageCode (const std::string &languageCode)
465 {
466 // be sure supported languages are initialized
467 initLanguages();
468
469 std::string lang = NLMISC::toLowerAscii(languageCode);
470
471 // specified language is really a code (2 characters)
472 if (lang.length() == 2)
473 {
474 // check if language code is supported
475 for(uint i = 0; i < _LanguageCodes.size(); ++i)
476 {
477 std::string code = NLMISC::toLowerAscii(_LanguageCodes[i]);
478
479 if (lang == code)
480 {
481 // found, so use it
482 _SystemLanguageCode = lang;
483 return true;
484 }
485 }
486 }
487 // specified language is something else
488 else
489 {
490 // check if language name is supported
491 for(uint i = 0; i < _LanguageNames.size(); ++i)
492 {
493 std::string name = NLMISC::toLowerAscii(_LanguageNames[i]);
494
495 if (name == lang)
496 {
497 // found, so use its code
498 _SystemLanguageCode = _LanguageCodes[i];
499 return true;
500 }
501 }
502 }
503
504 return false;
505 }
506
507 void CI18N::removeCComment(ucstring &commentedString)
508 {
509 ucstring temp;
510 temp.reserve(commentedString.size());
511 ucstring::const_iterator first(commentedString.begin()), last(commentedString.end());
512 for (;first != last; ++first)
513 {
514 temp.push_back(*first);
515 if (*first == '[')
516 {
517 // no comment inside string literal
518 while (++first != last)
519 {
520 temp.push_back(*first);
521 if (*first == ']')
522 break;
523 }
524 }
525 else if (*first == '/')
526 {
527 // start of comment ?
528 ++first;
529 if (first != last && *first == '/')
530 {
531 temp.resize(temp.size()-1);
532 // one line comment, skip until end of line
533 while (first != last && *first != '\n')
534 ++first;
535 }
536 else if (first != last && *first == '*')
537 {
538 temp.resize(temp.size()-1);
539 // start of multi line comment, skip until we found '*/'
540 while (first != last && !(*first == '*' && (first+1) != last && *(first+1) == '/'))
541 ++first;
542 // skip the closing '/'
543 if (first != last)
544 ++first;
545 }
546 else
547 {
548 temp.push_back(*first);
549 }
550 }
551 }
552 commentedString.swap(temp);
553 }
554
555 void CI18N::skipWhiteSpace(ucstring::const_iterator &it, ucstring::const_iterator &last, ucstring *storeComments, bool newLineAsWhiteSpace)
556 {
557 while (it != last &&
558 (
559 (*it == 0xa && newLineAsWhiteSpace)
560 || (*it == 0xd && newLineAsWhiteSpace)
561 || *it == ' '
562 || *it == '\t'
563 || (storeComments && *it == '/' && it+1 != last && *(it+1) == '/')
564 || (storeComments && *it == '/' && it+1 != last && *(it+1) == '*')
565 ))
566 {
567 if (storeComments && *it == '/' && it+1 != last && *(it+1) == '/')
568 {
569 // found a one line C comment. Store it until end of line.
570 while (it != last && (*it != '\n' && *it != '\r'))
571 storeComments->push_back(*it++);
572
573 // store the final '\n'
574 if (it != last)
575 storeComments->push_back('\n');
576 }
577 else if (storeComments && *it == '/' && it+1 != last && *(it+1) == '*')
578 {
579 // found a multi line C++ comment. store until we found the closing '*/'
580 while (it != last && !(*it == '*' && it + 1 != last && *(it + 1) == '/'))
581 {
582 // don't put \r
583 if (*it == '\r')
584 {
585 // skip it
586 ++it;
587 }
588 else
589 {
590 storeComments->push_back(*it++);
591 }
592 }
593
594 // store the final '*'
595 if (it != last)
596 storeComments->push_back(*it++);
597
598 // store the final '/'
599 if (it != last)
600 storeComments->push_back(*it++);
601
602 // and a new line.
603 storeComments->push_back('\n');
604 }
605 else
606 {
607 // just skip white space or don't store comments
608 ++it;
609 }
610 }
611 }
612
613 bool CI18N::parseLabel(ucstring::const_iterator &it, ucstring::const_iterator &last, string &label)
614 {
615 ucstring::const_iterator rewind = it;
616 label.erase();
617
618 // first char must be A-Za-z@_
619 if (it != last &&
620 (
621 (*it >= '0' && *it <= '9')
622 || (*it >= 'A' && *it <= 'Z')
623 || (*it >= 'a' && *it <= 'z')
624 || (*it == '_')
625 || (*it == '@')
626 )
627 )
628 label.push_back(char(*it++));
629 else
630 {
631 it = rewind;
632 return false;
633 }
634
635 // other char must be [0-9A-Za-z@_]*
636 while (it != last &&
637 (
638 (*it >= '0' && *it <= '9')
639 || (*it >= 'A' && *it <= 'Z')
640 || (*it >= 'a' && *it <= 'z')
641 || (*it == '_')
642 || (*it == '@')
643 )
644 )
645 label.push_back(char(*it++));
646
647 return true;
648 }
649
650 bool CI18N::parseMarkedString(ucchar openMark, ucchar closeMark, ucstring::const_iterator &it, ucstring::const_iterator &last, ucstring &result, uint32 *lineCounter, bool allowNewline)
651 {
652 result.erase();
653
654 // parse a string delimited by the specified opening and closing mark
655 if (it != last && *it == openMark)
656 {
657 ++it;
658
659 while (it != last && *it != closeMark && (allowNewline || *it != '\n'))
660 {
661 // ignore tab, new lines and line feed
662 if (*it == openMark)
663 {
664 nlwarning("I18N: Found a non escaped openmark %c in a delimited string (Delimiters : '%c' - '%c')", char(openMark), char(openMark), char(closeMark));
665 return false;
666 }
667 if (*it == '\t'
668 || (*it == '\n' && allowNewline)
669 || *it == '\r')
670 ++it;
671 else if (*it == '\\' && it+1 != last && *(it+1) != '\\')
672 {
673 ++it;
674 // this is an escape sequence !
675 switch(*it)
676 {
677 case 't':
678 result.push_back('\t');
679 break;
680 case 'n':
681 result.push_back('\n');
682 break;
683 case 'd':
684 // insert a delete
685 result.push_back(8);
686 break;
687 default:
688 // escape the close mark ?
689 if(*it == closeMark)
690 result.push_back(closeMark);
691 // escape the open mark ?
692 else if(*it == openMark)
693 result.push_back(openMark);
694 else
695 {
696 nlwarning("I18N: Ignoring unknown escape code \\%c (char value : %u)", char(*it), *it);
697 return false;
698 }
699 }
700 ++it;
701 }
702 else if (*it == '\\' && it+1 != last && *(it+1) == '\\')
703 {
704 // escape the \ char
705 ++it;
706 result.push_back(*it);
707 ++it;
708 }
709 else
710 {
711 if (*it == '\n' && lineCounter != NULL)
712 // update line counter
713 ++(*lineCounter);
714
715 result.push_back(*it++);
716 }
717 }
718
719 if (it == last || *it != closeMark)
720 {
721 nlwarning("I18N: Missing end of delimited string (Delimiters : '%c' - '%c')", char(openMark), char(closeMark));
722 return false;
723 }
724 else
725 ++it;
726 }
727 else
728 {
729 nlwarning("I18N: Malformed or non existent delimited string (Delimiters : '%c' - '%c')", char(openMark), char(closeMark));
730 return false;
731 }
732
733 return true;
734 }
735
736 void CI18N::readTextFile(const string &filename,
737 ucstring &result,
738 bool fileLookup,
739 bool preprocess,
740 TLineFormat lineFmt,
741 bool warnIfIncludesNotFound)
742 {
743 // create the read context
744 TReadContext readContext;
745
746 // call the inner function
747 _readTextFile(filename, result, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
748
749 if (!readContext.IfStack.empty())
750 {
751 nlwarning("Preprocess: Missing %u closing #endif after parsing %s", (uint)readContext.IfStack.size(), filename.c_str() );
752 }
753 }
754
755 bool CI18N::matchToken(const char* token, ucstring::const_iterator &it, ucstring::const_iterator end)
756 {
757 ucstring::const_iterator rewind = it;
758 skipWhiteSpace(it, end, NULL, false);
759 while (it != end && *token != 0 && *it == *token)
760 {
761 ++it;
762 ++token;
763 }
764
765 if (*token == 0)
766 {
767 // we fund the token
768 return true;
769 }
770
771 // not found
772 it = rewind;
773 return false;
774 }
775
776 void CI18N::skipLine(ucstring::const_iterator &it, ucstring::const_iterator end, uint32 &lineCounter)
777 {
778 while (it != end && *it != '\n')
779 ++it;
780
781 if (it != end)
782 {
783 ++lineCounter;
784 ++it;
785 }
786 }
787
788 void CI18N::_readTextFile(const string &filename,
789 ucstring &result,
790 bool fileLookup,
791 bool preprocess,
792 TLineFormat lineFmt,
793 bool warnIfIncludesNotFound,
794 TReadContext &readContext)
795 {
796 string fullName;
797 if (fileLookup)
798 fullName = CPath::lookup(filename, false,warnIfIncludesNotFound);
799 else
800 fullName = filename;
801
802 if (fullName.empty())
803 return;
804
805 // If ::lookup is used, the file can be in a bnp and CFile::fileExists fails.
806 bool isInBnp = fullName.find('@') != string::npos;
807 if (!isInBnp && !CFile::fileExists(fullName))
808 {
809 nlwarning("CI18N::readTextFile : file '%s' does not exist, returning empty string", fullName.c_str());
810 return;
811 }
812
813 NLMISC::CIFile file(fullName);
814
815 // Fast read all the text in binary mode.
816 string text;
817 text.resize(file.getFileSize());
818 if (file.getFileSize() > 0)
819 file.serialBuffer((uint8*)(&text[0]), (uint)text.size());
820
821 // Transform the string in ucstring according to format header
822 if (!text.empty())
823 readTextBuffer((uint8*)&text[0], (uint)text.size(), result);
824
825 if (preprocess)
826 {
827 // a string to old the result of the preprocess
828 ucstring final;
829 // make rooms to reduce allocation cost
830 final.reserve(raiseToNextPowerOf2((uint)result.size()));
831
832 // parse the file, looking for preprocessor command.
833 ucstring::const_iterator it(result.begin()), end(result.end());
834
835 // input line counter
836 uint32 currentLine = 1;
837
838 // set the current file and line info
839 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
840
841 while (it != end)
842 {
843 // remember the begin of the line
844 ucstring::const_iterator beginOfLine = it;
845
846 // advance in the line, looking for a preprocessor command
847 skipWhiteSpace(it, end, NULL, false);
848
849 if (it != end && *it == '#')
850 {
851 // skip the '#' symbol
852 ++it;
853 // we found a preprocessor command !
854 skipWhiteSpace(it, end, NULL, false);
855
856 if (matchToken("include", it, end))
857 {
858 if (readContext.IfStack.empty() || readContext.IfStack.back())
859 {
860 // we have an include command
861 skipWhiteSpace(it, end, NULL, false);
862
863 // read the file name between quote
864 ucstring str;
865 breakable
866 {
867 if (!parseMarkedString(ucchar('\"'), ucchar('\"'), it, end, str, &currentLine, false))
868 {
869 nlwarning("Preprocess: In file %s(%u) : Error parsing include file command", filename.c_str(), currentLine);
870
871 break;
872 }
873 else
874 {
875 // ok, read the subfile
876 string subFilename = str.toString();
877
878 // check is file exist
879 if (!CFile::fileExists(subFilename))
880 {
881 // look for the file relative to current file
882 subFilename = CFile::getPath(filename)+subFilename;
883 if (!CFile::fileExists(subFilename))
884 {
885 // the include file is not found, issue a warning
886 nlwarning("Preprocess: In file %s(%u) : Cannot include file '%s'",
887 filename.c_str(), currentLine,
888 str.toString().c_str());
889
890 break;
891 }
892 }
893
894 nlinfo("Preprocess: In file %s(%u) : Including '%s'",
895 filename.c_str(), currentLine,
896 subFilename.c_str());
897
898 ucstring inserted;
899 _readTextFile(subFilename, inserted, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
900 final += inserted;
901 }
902 }
903 // advance to next line
904 skipLine(it, end, currentLine);
905 // reset filename and line counter
906 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
907 }
908 }
909 else if (matchToken("optional", it, end))
910 {
911 if (readContext.IfStack.empty() || readContext.IfStack.back())
912 {
913 // we have an optional include command
914 skipWhiteSpace(it, end, NULL, false);
915
916 // read the file name between quote
917 ucstring str;
918 breakable
919 {
920 if (!parseMarkedString('\"', '\"', it, end, str, &currentLine, false))
921 {
922 nlwarning("Preprocess: In file %s(%u) : Error parsing optional file command", filename.c_str(), currentLine);
923
924 break;
925 }
926 else
927 {
928 // ok, read the subfile
929 string subFilename = str.toString();
930
931 // check is file exist
932 if (!CFile::fileExists(subFilename))
933 {
934 // look for the file relative to current file
935 subFilename = CFile::getPath(filename)+subFilename;
936 if (!CFile::fileExists(subFilename))
937 {
938 // not found but optional, only emit a debug log
939 // the include file is not found, issue a warning
940 nldebug("Preprocess: In file %s(%u) : Cannot include optional file '%s'",
941 filename.c_str(), currentLine,
942 str.toString().c_str());
943
944 break;
945 }
946 }
947
948 nlinfo("Preprocess: In file %s(%u) : Including optional '%s'",
949 filename.c_str(), currentLine,
950 subFilename.c_str());
951
952 ucstring inserted;
953 _readTextFile(subFilename, inserted, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
954 final += inserted;
955 }
956 }
957 // advance to next line
958 skipLine(it, end, currentLine);
959 // reset filename and line counter
960 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
961 }
962 }
963 else if (matchToken("define", it, end))
964 {
965 if (readContext.IfStack.empty() || readContext.IfStack.back())
966 {
967 skipWhiteSpace(it, end, NULL, false);
968
969 string label;
970 if (parseLabel(it, end, label))
971 {
972 if (readContext.Defines.find(label) != readContext.Defines.end())
973 {
974 nlinfo("Preprocess: In file %s(%u) : symbol '%s' already defined",
975 filename.c_str(), currentLine,
976 label.c_str());
977 }
978 else
979 {
980 readContext.Defines.insert(label);
981 }
982 }
983 else
984 {
985 nlwarning("Preprocess: In file %s(%u) : Error parsing #define command", filename.c_str(), currentLine);
986 }
987
988 // advance to next line
989 skipLine(it, end, currentLine);
990 // update filename and line number
991 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
992 }
993 }
994 else if (matchToken("ifdef", it, end))
995 {
996 if (readContext.IfStack.empty() || readContext.IfStack.back())
997 {
998 skipWhiteSpace(it, end, NULL, false);
999 string label;
1000 if (parseLabel(it, end, label))
1001 {
1002 if (readContext.Defines.find(label) != readContext.Defines.end())
1003 {
1004 // symbol defined, push a true
1005 readContext.IfStack.push_back(true);
1006 }
1007 else
1008 {
1009 // symbol not defines, push a false
1010 readContext.IfStack.push_back(false);
1011 }
1012 }
1013 else
1014 {
1015 nlwarning("Preprocess: In file %s(%u) : Error parsing #ifdef command", filename.c_str(), currentLine);
1016 }
1017
1018 // advance to next line
1019 skipLine(it, end, currentLine);
1020 // update filename and line number
1021 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
1022 }
1023 else
1024 {
1025 // just push to false
1026 readContext.IfStack.push_back(false);
1027
1028 skipLine(it, end, currentLine);
1029 }
1030 }
1031 else if (matchToken("ifndef", it, end))
1032 {
1033 if (readContext.IfStack.empty() || readContext.IfStack.back())
1034 {
1035 skipWhiteSpace(it, end, NULL, false);
1036 string label;
1037 if (parseLabel(it, end, label))
1038 {
1039 if (readContext.Defines.find(label) == readContext.Defines.end())
1040 {
1041 // symbol defined, push a true
1042 readContext.IfStack.push_back(true);
1043 }
1044 else
1045 {
1046 // symbol not defines, push a false
1047 readContext.IfStack.push_back(false);
1048 }
1049 }
1050 else
1051 {
1052 nlwarning("Preprocess: In file %s(%u) : Error parsing #ifndef command", filename.c_str(), currentLine);
1053 }
1054
1055 // advance to next line
1056 skipLine(it, end, currentLine);
1057 // update filename and line number
1058 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
1059 }
1060 else
1061 {
1062 // just push to false
1063 readContext.IfStack.push_back(false);
1064
1065 skipLine(it, end, currentLine);
1066 }
1067 }
1068 else if (matchToken("endif", it, end))
1069 {
1070 bool previous = false;
1071 if (readContext.IfStack.empty())
1072 {
1073 nlwarning("Preprocess: In file %s(%u) : Error found '#endif' without matching #if", filename.c_str(), currentLine);
1074 }
1075 else
1076 {
1077 previous = readContext.IfStack.back();
1078
1079 readContext.IfStack.pop_back();
1080 }
1081 skipLine(it, end, currentLine);
1082
1083 if (!previous && (readContext.IfStack.empty() || readContext.IfStack.back()))
1084 {
1085 // end of ignored file part, restore the file and line number
1086 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
1087 }
1088 // update filename and line number
1089 // final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
1090 }
1091 else
1092 {
1093 // unrecognized command, ignore line
1094 nlwarning("Preprocess: In file %s(%u) : Error unrecognized preprocessor command",
1095 filename.c_str(), currentLine);
1096
1097 skipLine(it, end, currentLine);
1098 // update filename and line number
1099 final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
1100 }
1101 }
1102 else
1103 {
1104 // normal line
1105 skipLine(it, end, currentLine);
1106
1107 if (readContext.IfStack.empty() || readContext.IfStack.back())
1108 {
1109 // copy the line to the final string
1110 final.append(beginOfLine, it);
1111 }
1112 }
1113 }
1114
1115 // set the result with the preprocessed content
1116 result.swap(final);
1117 }
1118
1119 // apply line delimiter conversion if needed
1120 if (lineFmt != LINE_FMT_NO_CARE)
1121 {
1122 if (lineFmt == LINE_FMT_LF)
1123 {
1124 // we only want \n
1125 // easy, just remove or replace any \r code
1126 string::size_type pos;
1127 string::size_type lastPos = 0;
1128 ucstring temp;
1129 // reserve some place to reduce re-allocation
1130 temp.reserve(result.size() +result.size()/10);
1131
1132 // look for the first \r
1133 pos = result.find('\r');
1134 while (pos != string::npos)
1135 {
1136 if (pos < result.size()-1 && result[pos+1] == '\n')
1137 {
1138 temp.append(result.begin()+lastPos, result.begin()+pos);
1139 pos += 1;
1140 }
1141 else
1142 {
1143 temp.append(result.begin()+lastPos, result.begin()+pos);
1144 temp[temp.size()-1] = '\n';
1145 }
1146
1147 lastPos = pos;
1148 // look for next \r
1149 pos = result.find('\r', pos);
1150 }
1151
1152 // copy the rest
1153 temp.append(result.begin()+lastPos, result.end());
1154
1155 result.swap(temp);
1156 }
1157 else if (lineFmt == LINE_FMT_CRLF)
1158 {
1159 // need to replace simple '\n' or '\r' with a '\r\n' double
1160 string::size_type pos = 0;
1161 string::size_type lastPos = 0;
1162
1163 ucstring temp;
1164 // reserve some place to reduce re-allocation
1165 temp.reserve(result.size() +result.size()/10);
1166
1167
1168 // first loop with the '\r'
1169 pos = result.find('\r', pos);
1170 while (pos != string::npos)
1171 {
1172 if (pos >= result.size()-1 || result[pos+1] != '\n')
1173 {
1174 temp.append(result.begin()+lastPos, result.begin()+pos+1);
1175 temp += '\n';
1176 lastPos = pos+1;
1177 }
1178 // skip this char
1179 pos++;
1180
1181 // look the next '\r'
1182 pos = result.find('\r', pos);
1183 }
1184
1185 // copy the rest
1186 temp.append(result.begin()+lastPos, result.end());
1187 result.swap(temp);
1188
1189 temp.clear();
1190
1191 // second loop with the '\n'
1192 pos = 0;
1193 lastPos = 0;
1194 pos = result.find('\n', pos);
1195 while (pos != string::npos)
1196 {
1197 if (pos == 0 || result[pos-1] != '\r')
1198 {
1199 temp.append(result.begin()+lastPos, result.begin()+pos);
1200 temp += '\r';
1201 temp += '\n';
1202 lastPos = pos+1;
1203 }
1204 // skip this char
1205 pos++;
1206
1207 pos = result.find('\n', pos);
1208 }
1209
1210 // copy the rest
1211 temp.append(result.begin()+lastPos, result.end());
1212 result.swap(temp);
1213 }
1214 }
1215 }
1216
1217 void CI18N::readTextBuffer(uint8 *buffer, uint size, ucstring &result)
1218 {
1219 static uint8 utf16Header[] = { 0xffu, 0xfeu };
1220 static uint8 utf16RevHeader[] = { 0xfeu, 0xffu };
1221 static uint8 utf8Header[] = { 0xefu, 0xbbu, 0xbfu };
1222
1223 if (size>=3 &&
1224 buffer[0]==utf8Header[0] &&
1225 buffer[1]==utf8Header[1] &&
1226 buffer[2]==utf8Header[2]
1227 )
1228 {
1229 // remove utf8 header
1230 buffer+= 3;
1231 size-=3;
1232 string text((char*)buffer, size);
1233 result.fromUtf8(text);
1234 }
1235 else if (size>=2 &&
1236 buffer[0]==utf16Header[0] &&
1237 buffer[1]==utf16Header[1]
1238 )
1239 {
1240 // remove utf16 header
1241 buffer+= 2;
1242 size-= 2;
1243 // check pair number of bytes
1244 nlassert((size & 1) == 0);
1245 // and do manual conversion
1246 uint16 *src = (uint16*)(buffer);
1247 result.resize(size/2);
1248 for (uint j=0; j<result.size(); j++)
1249 result[j]= *src++;
1250 }
1251 else if (size>=2 &&
1252 buffer[0]==utf16RevHeader[0] &&
1253 buffer[1]==utf16RevHeader[1]
1254 )
1255 {
1256 // remove utf16 header
1257 buffer+= 2;
1258 size-= 2;
1259 // check pair number of bytes
1260 nlassert((size & 1) == 0);
1261 // and do manual conversion
1262 uint16 *src = (uint16*)(buffer);
1263 result.resize(size/2);
1264 uint j;
1265 for (j=0; j<result.size(); j++)
1266 result[j]= *src++;
1267 // Reverse byte order
1268 for (j=0; j<result.size(); j++)
1269 {
1270 uint8 *pc = (uint8*) &result[j];
1271 swap(pc[0], pc[1]);
1272 }
1273 }
1274 else
1275 {
1276 // all text files without BOM are now parsed as UTF-8 by default
1277 string text((char*)buffer, size);
1278 result.fromUtf8(text);
1279 }
1280 }
1281
1282 void CI18N::writeTextFile(const string filename, const ucstring &content, bool utf8)
1283 {
1284 COFile file(filename);
1285
1286 if (!utf8)
1287 {
1288 // write the Unicode 16 bits tag
1289 uint16 unicodeTag = 0xfeff;
1290 file.serial(unicodeTag);
1291
1292 uint i;
1293 for (i=0; i<content.size(); ++i)
1294 {
1295 uint16 c = content[i];
1296 file.serial(c);
1297 }
1298 }
1299 else
1300 {
1301 static char utf8Header[] = {char(0xef), char(0xbb), char(0xbf), 0};
1302
1303 string str = encodeUTF8(content);
1304 // add the UTF-8 'not official' header
1305 str = utf8Header + str;
1306
1307 uint i;
1308 for (i=0; i<str.size(); ++i)
1309 {
1310 file.serial(str[i]);
1311 }
1312 }
1313 }
1314
1315 ucstring CI18N::makeMarkedString(ucchar openMark, ucchar closeMark, const ucstring &text)
1316 {
1317 ucstring ret;
1318
1319 ret.push_back(openMark);
1320
1321 ucstring::const_iterator first(text.begin()), last(text.end());
1322 for (; first != last; ++first)
1323 {
1324 if (*first == '\n')
1325 {
1326 ret += '\\';
1327 ret += 'n';
1328 }
1329 else if (*first == '\t')
1330 {
1331 ret += '\\';
1332 ret += 't';
1333 }
1334 else if (*first == closeMark)
1335 {
1336 // escape the embedded closing mark
1337 ret += '\\';
1338 ret += closeMark;
1339 }
1340 else
1341 {
1342 ret += *first;
1343 }
1344 }
1345
1346 ret += closeMark;
1347
1348 return ret;
1349 }
1350
1351 string CI18N::encodeUTF8(const ucstring &str)
1352 {
1353 return str.toUtf8();
1354 }
1355
1356 /* UTF-8 conversion table
1357 U-00000000 - U-0000007F: 0xxxxxxx
1358 U-00000080 - U-000007FF: 110xxxxx 10xxxxxx
1359 U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
1360 // not used as we convert from 16 bits unicode
1361 U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
1362 U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
1363 U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
1364 */
1365
1366 uint64 CI18N::makeHash(const ucstring &str)
1367 {
1368 // we do at least 8 pass on each result byte
1369 if (str.empty())
1370 return 0;
1371 const uint32 MIN_TURN = 8*8;
1372 uint64 hash = 0;
1373 uint8 *ph = (uint8*)&hash;
1374 uint8 *pc = (uint8*)str.data();
1375
1376 uint nbLoop = max(uint32(str.size()*2), MIN_TURN);
1377 uint roll = 0;
1378
1379 for (uint i=0; i<nbLoop; ++i)
1380 {
1381 ph[(i/2) & 0x7] = uint8((ph[(i/2) & 0x7] + (pc[i%(str.size()*2)] << roll)) & 0xff);
1382 ph[(i/2) & 0x7] = uint8((ph[(i/2) & 0x7] + (pc[i%(str.size()*2)] >> (8-roll))) & 0xff);
1383
1384 roll++;
1385 roll &= 0x7;
1386 }
1387
1388 return hash;
1389 }
1390
1391 // convert a hash value to a readable string
1392 string CI18N::hashToString(uint64 hash)
1393 {
1394 char temp[] = "0011223344556677";
1395 sprintf(temp, "%08X%08X", (uint32)(hash & 0xffffffff), (uint32)(hash >> 32));
1396
1397 return string(temp);
1398 }
1399
1400 // fast convert a hash value to a ucstring
1401 void CI18N::hashToUCString(uint64 hash, ucstring &dst)
1402 {
1403 static ucchar cvtTable[]= {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
1404
1405 dst.resize(16);
1406 for(sint i=15;i>=0;i--)
1407 {
1408 // Must decal dest of 8, cause of hashToString code (Little Endian)
1409 dst[(i+8)&15]= cvtTable[hash&15];
1410 hash>>=4;
1411 }
1412 }
1413
1414 // convert a readable string into a hash value.
1415 uint64 CI18N::stringToHash(const string &str)
1416 {
1417 nlassert(str.size() == 16);
1418 uint32 low, high;
1419
1420 string sl, sh;
1421 sh = str.substr(0, 8);
1422 sl = str.substr(8, 8);
1423
1424 sscanf(sh.c_str(), "%08X", &high);
1425 sscanf(sl.c_str(), "%08X", &low);
1426
1427 uint64 hash;
1428
1429 memcpy(&hash, &high, sizeof(high));
1430 memcpy((uint32*)&hash + 1, &low, sizeof(low));
1431
1432 return hash;
1433 }
1434
1435 } // namespace NLMISC