?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'fixes' into main/rendor-staging
[ryzomcore.git] / ryzom / tools / translation_tools / main.cpp
blobdc91079f3156aff79992849b6248da7060be794f
1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
2 // Copyright (C) 2010 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2014-2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
6 //
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
11 //
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
16 //
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20
21
22
23 /** This tool is used to managed translation file.
24 * I work with two different file format :
25 * - phrase file witch contain a complex grammar description
26 * - string file withc contain only pair of identifier / string value.
27 *
28 * This tool can do 6 different work :
29 * - make diff string file file for each language from a reference string file.
30 *
31 * - merge the translated diff string file into there respective string file after
32 * translation
33 *
34 * - make diff phrase file for each language from a reference phrase file
35 *
36 * - merge the translated diff phrase file into there respective phrase file after
37 * translation
38 *
39 * - make clause diff for each language by examining phrase files. Add comments
40 * in the diff files for phrase parameter information.
41 *
42 * - merge clause diff in all the clause file.
43 *
44 * - remove "\*OLDVALUE: \*\/" from clause file or phrase file
45 *
46 *
47 * Before invocation, you must be in the translation repository (see localisation_system_in_ryzom.doc)
48 * Invocation should be as folow :
49 * trans_tool make_string_diff
50 * trans_tool merge_string_diff
51 * trans_tool make_words_diff
52 * trans_tool merge_words_diff
53 * trans_tool make_phrase_diff
54 * trans_tool merge_phrase_diff
55 * trans_tool make_clause_diff
56 * trans_tool merge_clause_diff
57 * trans_tool clean_string_diff
58 * trans_tool clean_words_diff
59 * trans_tool clean_clause_diff
60 * trans_tool clean_phrase_diff
61 * trans_tool make_phrase_diff_old
62 * trans_tool merge_phrase_diff_old
63 * trans_tool forget_phrase_diff
64 * trans_tool update_phrase_work
65 * trans_tool inject_clause
66 * trans_tool sort_trans_phrase
67 * trans_tool make_worksheet_diff
68 * trans_tool merge_worksheet_diff
69 * trans_tool crop_lines
70 * trans_tool extract_bot_names
71 * trans_tool extract_new_sheet_names
72
73 */
74
75 #include "nel/misc/app_context.h"
76 #include "nel/misc/i18n.h"
77 #include "nel/misc/common.h"
78 #include "nel/misc/file.h"
79 #include "nel/misc/path.h"
80 #include "nel/misc/diff_tool.h"
81 #include "nel/misc/algo.h"
82 #include <vector>
83 #include <list>
84 #include <algorithm>
85 #include <stdio.h>
86 #include <time.h>
87 #include <iterator>
88
89 using namespace std;
90 using namespace NLMISC;
91 using namespace STRING_MANAGER;
92
93 int extractBotNames(int argc, char *argv[]);
94 int extractNewSheetNames(int argc, char *argv[]);
95 const std::string addDir("work/");
96 const std::string diffDir("diff/");
97 const std::string transDir("translated/");
98 const std::string historyDir("history/");
99
100 string diffVersion;
101
102 #ifdef NL_DEBUG
103 # define LOG nldebug
104 #else
105 # define LOG printf
106 #endif
107
108 enum TDiffCommand
109 {
110 diff_none,
111 diff_add,
112 diff_changed,
113 diff_removed,
114 diff_swap,
115 diff_keep
116 };
117
118 struct TDiffInfo
119 {
120 TDiffCommand Command;
121 uint Index1;
122 uint Index2;
123 };
124
125 /// Store the list of language extracted from the languages.txt file
126 vector<string> Languages;
127
128
129 void showUsage(char *exeName)
130 {
131 LOG("%s usage : \n", exeName);
132 LOG(" %s <command> [<filename>]\n", exeName);
133 LOG(" Where command can be :\n");
134 LOG(" make_string_diff\n");
135 LOG(" merge_string_diff\n");
136 LOG(" clean_string_diff\n");
137 LOG(" make_phrase_diff\n");
138 LOG(" merge_phrase_diff\n");
139 LOG(" clean_phrase_diff\n");
140 LOG(" make_clause_diff\n");
141 LOG(" merge_clause_diff\n");
142 LOG(" clean_clause_diff\n");
143 LOG(" make_phrase_diff_old\n");
144 LOG(" merge_phrase_diff_old\n");
145 LOG(" forget_phrase_diff\n");
146 LOG(" inject_clause\n");
147 LOG(" sort_trans_phrase\n");
148 LOG(" make_worksheet_diff <filename>\n");
149 LOG(" merge_worksheet_diff <filename>\n");
150 LOG(" crop_lines <filename> <nbLines>\n");
151 LOG(" extract_bot_names [-r]\n");
152 LOG(" extract_new_sheet_names [-r]\n");
153 LOG("\n");
154 LOG("Language code are ISO 639-2 + optionally ISO 3166 country code.\n");
155 LOG("Reference language is always the first language in languages.txt\n");
156 }
157
158 void verifyVersion(ucstring& doc, int versionId)
159 {
160 ucstring version1("// DIFF_VERSION 1\n");
161 ucstring::size_type version1Size = version1.size();
162 ucstring version2("// DIFF_VERSION 2\n");
163 ucstring::size_type version2Size = version2.size();
164
165 switch (versionId)
166 {
167 case 1:
168 if (doc.size() < version1Size|| doc.substr(0, version1Size) != version1 )
169 {
170 nlerror("Loading wrong diff version");
171 nlassert(0);
172 }
173 doc = doc.substr(version1Size);
174 break;
175
176 case 2:
177 if (doc.size() < version2Size || doc.substr(0, version2Size) != version2 )
178 {
179 nlerror("Loading wrong diff version");
180 nlassert(0);
181 }
182 doc = doc.substr(version2Size);
183 break;
184
185 default:
186 nlassert(0);
187 }
188
189
190 }
191 bool readPhraseFile1(const std::string &filename, vector<TPhrase> &phrases, bool forceRehash)
192 {
193 ucstring doc;
194
195 CI18N::readTextFile(filename, doc, false, false, CI18N::LINE_FMT_LF);
196 verifyVersion(doc, 1);
197 return readPhraseFileFromString(doc, filename, phrases, forceRehash);
198 }
199
200 bool readPhraseFile2(const std::string &filename, vector<TPhrase> &phrases, bool forceRehash)
201 {
202 ucstring doc;
203
204 CI18N::readTextFile(filename, doc, false, false, CI18N::LINE_FMT_LF);
205 verifyVersion(doc, 2);
206 return readPhraseFileFromString(doc, filename, phrases, forceRehash);
207 }
208
209
210
211 void getPathContentFiltered(const string &baseName, const string &ext, vector<string> &result)
212 {
213 CPath::getPathContent(diffDir, false, false, true, result);
214
215 uint i;
216 for (i=0; i<result.size(); ++i)
217 {
218 if (result[i].find(baseName) != 0 || result[i].rfind(ext) != result[i].size()-ext.size())
219 {
220 // remove it from the list
221 result.erase(result.begin()+i);
222 --i;
223 }
224 }
225 }
226
227
228 bool parseDiffCommandFromComment(const ucstring &comments, TDiffInfo &diffInfo)
229 {
230 ucstring::size_type pos = comments.find(ucstring("DIFF "));
231 if (pos == string::npos)
232 return false;
233
234 pos += 5;
235 ucstring::const_iterator it(comments.begin()+pos), last(comments.end());
236
237 string commandStr;
238 if (!CI18N::parseLabel(it, last, commandStr))
239 return false;
240
241 CI18N::skipWhiteSpace(it, last);
242 if (commandStr == "SWAP")
243 diffInfo.Command = diff_swap;
244 else if (commandStr == "ADD")
245 diffInfo.Command = diff_add;
246 else if (commandStr == "CHANGED")
247 diffInfo.Command = diff_changed;
248 else if (commandStr == "REMOVED")
249 diffInfo.Command = diff_removed;
250 else if (commandStr == "KEEP")
251 diffInfo.Command = diff_keep;
252 else
253 {
254 nlwarning("Invalid diff command '%s'", commandStr.c_str());
255 diffInfo.Command = diff_none;
256 return false;
257 }
258
259 CI18N::skipWhiteSpace(it, last);
260 // ok, parse the index.
261 string indexStr;
262 if (!CI18N::parseLabel(it, last, indexStr))
263 return false;
264
265 NLMISC::fromString(indexStr, diffInfo.Index1);
266
267 if (diffInfo.Command == diff_swap)
268 {
269 CI18N::skipWhiteSpace(it, last);
270 if (!CI18N::parseLabel(it, last, indexStr))
271 return false;
272
273 NLMISC::fromString(indexStr, diffInfo.Index2);
274 }
275 return true;
276 }
277
278
279 /// Read the languages.txt file.
280 int readLanguages()
281 {
282 // read the language list file
283 ucstring f;
284 CI18N::readTextFile("languages.txt", f);
285 string lang;
286
287 if (f.empty())
288 {
289 LOG("Error : the file languages.txt is missing or empty !\n");
290 return 1;
291 }
292
293 ucstring::const_iterator first(f.begin()), last(f.end());
294 while (first != last)
295 {
296 CI18N::skipWhiteSpace(first, last);
297
298 // read a language code
299 while (*first != ' ' && *first != '\n' && *first != '\r' && *first != '\t')
300 lang += char(*first++);
301
302 if (!lang.empty())
303 {
304 LOG("Adding language %s\n", lang.c_str());
305 Languages.push_back(lang);
306 lang.erase();
307 }
308
309 CI18N::skipWhiteSpace(first, last);
310 }
311 if (Languages.empty())
312 {
313 LOG("Error : the file languages.txt is empty !\n");
314 return 1;
315 }
316
317 LOG("Found %u language code\n", (uint) Languages.size());
318
319 return 0;
320 }
321
322
323 /*void appendToFile(const std::string &filename, const ucstring &text)
324 {
325 if (!CFile::fileExists(filename))
326 {
327 // create the new translatio file
328 CI18N::writeTextFile(filename, text);
329 }
330 else
331 {
332 // append to the existing file
333 FILE *fp = nlfopen(filename, "ab");
334
335 for (uint i=0; i<text.size(); ++i)
336 {
337 fputc(text[i] & 0xff, fp);
338 fputc((text[i]>>8) & 0xff, fp);
339 }
340
341 fclose(fp);
342 }
343 }
344 */
345
346
347 bool mergeStringDiff(vector<TStringInfo> &strings, const string &language, const string &baseName, const string &ext, bool onlyTranslated, bool archiveDiff = false)
348 {
349 vector<string> diffs;
350
351 getPathContentFiltered(diffDir+baseName+language+"_diff_", ext, diffs);
352
353 for (uint i=0; i<diffs.size(); ++i)
354 {
355 if (onlyTranslated)
356 {
357 // Check if the diff is translated
358 ucstring text;
359 CI18N::readTextFile(diffs[i], text, false, false, CI18N::LINE_FMT_LF);
360 if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
361 {
362 LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(diffs[i]).c_str());
363 for (i=i+1; i<diffs.size(); ++i)
364 LOG(" Merge of Diff file [%s] delayed.\n", CFile::getFilename(diffs[i]).c_str());
365 return true;
366 }
367 }
368
369 // we found a diff file for the addition file.
370 LOG("Adding %s diff as reference\n", diffs[i].c_str());
371 vector<TStringInfo> diff;
372 if (!loadStringFile(diffs[i], diff, false))
373 return false;
374
375 for (uint j=0; j<diff.size(); ++j)
376 {
377 /* TDiffCommand command;
378 uint index;
379 uint index2;
380 */
381 TDiffInfo diffInfo;
382 if (!parseDiffCommandFromComment(diff[j].Comments, diffInfo))
383 return false;
384
385 switch(diffInfo.Command)
386 {
387 case diff_swap:
388 nlassertex(diffInfo.Index1 < strings.size(), ("Index %u out of max Range %u", diffInfo.Index1, strings.size()));
389 nlassertex(diffInfo.Index2 < strings.size(), ("Index %u out of max Range %u", diffInfo.Index2, strings.size()));
390 swap(strings[diffInfo.Index1], strings[diffInfo.Index2]);
391 // remove the swap from the comments
392 diff[j].Comments = diff[j].Comments.substr(diff[j].Comments.find(nl)+nl.length());
393 if (!diff[j].Comments.empty())
394 j--;
395 break;
396 case diff_add:
397 nlassert(diffInfo.Index1 <= strings.size());
398 strings.insert(strings.begin()+diffInfo.Index1, diff[j]);
399 break;
400 case diff_changed:
401 nlassert(diffInfo.Index1 < strings.size());
402 strings[diffInfo.Index1] = diff[j];
403 break;
404 case diff_removed:
405 nlassert(diffInfo.Index1 < strings.size());
406 strings.erase(strings.begin()+diffInfo.Index1);
407 break;
408 case diff_keep:
409 nlassert(diffInfo.Index1 < strings.size());
410 strings[diffInfo.Index1].HashValue = diff[j].HashValue;
411 break;
412 default:
413 nlassert(false);
414 }
415 }
416
417 if (archiveDiff)
418 {
419 // move the diff file in the history dir
420 CFile::moveFile(historyDir+CFile::getFilename(diffs[i]), diffs[i]);
421 }
422 }
423
424 return true;
425 }
426
427
428 class CMakeStringDiff : CMakeDiff<TStringInfo, TStringDiffContext>::IDiffCallback
429 {
430 public:
431 void run(const vector<TStringInfo> &addition, vector<TStringInfo> &reference, vector<TStringInfo> &diff)
432 {
433 TStringDiffContext context(addition, reference, diff);
434
435 CMakeDiff<TStringInfo, TStringDiffContext> differ;
436 differ.makeDiff(this, context);
437 }
438
439 void onEquivalent(uint addIndex, uint refIndex, TStringDiffContext &context)
440 {
441 // nothing to do
442 }
443
444 void onAdd(uint addIndex, uint refIndex, TStringDiffContext &context)
445 {
446 TStringInfo si = context.Addition[addIndex];
447 char temp[1024];
448 sprintf(temp, "// DIFF ADD %u ", addIndex);
449 si.Comments = ucstring(temp) + nl + si.Comments;
450
451 nlinfo("Added %s at %u", si.Identifier.c_str(), addIndex);
452 context.Diff.push_back(si);
453 }
454
455 void onRemove(uint addIndex, uint refIndex, TStringDiffContext &context)
456 {
457 TStringInfo si = context.Reference[refIndex];
458 char temp[1024];
459 sprintf(temp, "// DIFF REMOVED %u ", addIndex);
460 // NB : on vire les commentaires car il pourrais contenir des merdes..
461 si.Comments = ucstring(temp) + nl;
462
463 nlinfo("Removed %s at %u", si.Identifier.c_str(), addIndex);
464 context.Diff.push_back(si);
465 }
466
467 void onChanged(uint addIndex, uint refIndex, TStringDiffContext &context)
468 {
469 TStringInfo si = context.Addition[addIndex];
470 char temp[1024];
471 sprintf(temp, "// DIFF CHANGED %u ", addIndex);
472 si.Comments = ucstring(temp) + nl + si.Comments;
473 si.Comments = si.Comments + ucstring("/* OLD VALUE : [") + context.Reference[refIndex].Text + "] */" + nl;
474
475 nlinfo("Changed %s at %u", si.Identifier.c_str(), addIndex);
476 context.Diff.push_back(si);
477 }
478
479 void onSwap(uint newIndex, uint refIndex, TStringDiffContext &context)
480 {
481 TStringInfo si;
482 char temp[1024];
483 sprintf(temp, "// DIFF SWAP %u %u (swaping %s and %s)", newIndex, refIndex, context.Reference[newIndex].Identifier.c_str(), context.Reference[refIndex].Identifier.c_str());
484 // sprintf(temp, "// DIFF SWAP %u %u", newIndex, refIndex);
485
486 si.Comments = ucstring(temp) + nl + nl;
487 context.Diff.push_back(si);
488 }
489 };
490
491
492 void makeStringDiff(const vector<TStringInfo> &addition, vector<TStringInfo> &reference, vector<TStringInfo> &diff)
493 {
494 // just building the object will to the job !
495 CMakeStringDiff differ;
496 differ.run(addition, reference, diff);
497
498 /*
499 // compare the reference an addition file, remove any equivalent strings.
500 uint addCount=0, refCount=0;
501
502 while (addCount < addition.size() || refCount < reference.size())
503 {
504 bool equal = true;
505 if (addCount != addition.size() && refCount != reference.size())
506 {
507 equal = addition[addCount].HashValue == reference[refCount].HashValue;
508 }
509
510 vector<TStringInfo>::iterator it;
511
512 if (addCount == addition.size()
513 ||
514 (
515 !equal
516 // && find_if(addition.begin()+addCount, addition.end(), TFindStringInfo(reference[refCount].Identifier)) == addition.end()
517 && find_if(addition.begin(), addition.end(), TFindStringInfo(reference[refCount].Identifier)) == addition.end()
518 )
519 )
520 {
521 // this can only be removed elements
522 TStringInfo si = reference[refCount];
523 char temp[1024];
524 sprintf(temp, "// DIFF REMOVED %u ", addCount);
525 // NB : on vire les commentaires car il pourrais contenir des merdes..
526 si.Comments = ucstring(temp) + nl;
527
528 nlinfo("Removed %s at %u", si.Identifier.c_str(), addCount);
529 diff.push_back(si);
530 ++refCount;
531 }
532 else if (refCount == reference.size()
533 ||
534 (
535 !equal
536 // && find_if(reference.begin()+refCount, reference.end(), TFindStringInfo(addition[addCount].Identifier)) == reference.end()
537 && find_if(reference.begin(), reference.end(), TFindStringInfo(addition[addCount].Identifier)) == reference.end()
538 )
539 )
540 {
541 // this can only be addition
542 TStringInfo si = addition[addCount];
543 char temp[1024];
544 sprintf(temp, "// DIFF ADD %u ", addCount);
545 si.Comments = ucstring(temp) + nl + si.Comments;
546
547 nlinfo("Added %s at %u", si.Identifier.c_str(), addCount);
548 diff.push_back(si);
549 ++addCount;
550 }
551 else if (addition[addCount].Identifier != reference[refCount].Identifier)
552 {
553 // swap two element.
554 vector<TStringInfo>::iterator it = find_if(reference.begin(), reference.end(), TFindStringInfo(addition[addCount].Identifier));
555 if (it == reference.end())
556 {
557 // addition
558 TStringInfo si = addition[addCount];
559 char temp[1024];
560 sprintf(temp, "// DIFF ADD %u ", addCount);
561 si.Comments = ucstring(temp) + nl + si.Comments;
562
563 nlinfo("Added %s at %u", si.Identifier.c_str(), addCount);
564 diff.push_back(si);
565 ++addCount;
566 }
567 else
568 {
569 nlassert(it != reference.begin()+refCount);
570
571 swap(*it, reference[refCount]);
572
573 TStringInfo si;
574 char temp[1024];
575 sprintf(temp, "// DIFF SWAP %u %u", it - reference.begin(), refCount);
576
577 si.Comments = ucstring(temp) + nl;
578 diff.push_back(si);
579 }
580 }
581 else if (addition[addCount].HashValue != reference[refCount].HashValue)
582 {
583 // changed element
584 TStringInfo si = addition[addCount];
585 char temp[1024];
586 sprintf(temp, "// DIFF CHANGED %u ", addCount);
587 si.Comments = ucstring(temp) + nl + si.Comments;
588 si.Comments = si.Comments + ucstring("// OLD VALUE : [") + reference[refCount].Text + ']' + nl;
589
590 nlinfo("Changed %s at %u", si.Identifier.c_str(), addCount);
591 diff.push_back(si);
592 ++refCount;
593 ++addCount;
594 }
595 else
596 {
597 // same entry
598 nlinfo("Same %s at %u", addition[addCount].Identifier.c_str(), addCount);
599 addCount++;
600 refCount++;
601 }
602 }
603 */
604 }
605
606 int makeStringDiff(int argc, char *argv[], const std::string &baseName)
607 {
608 // this will generate diff from 'addition' directory
609 // for the reference <lang>.uxt file
610 // with the same file in the 'translated' directory.
611
612 // NB : we use standard C file access because there are mutiple file with the same name in different place.
613
614 vector<TStringInfo> addition;
615
616 LOG("Generating string diffs\nLoading the working file for language %s\n", Languages[0].c_str());
617 // load the addition file
618 std::string addFile = baseName + Languages[0] + ".uxt";
619 if (!loadStringFile(addDir+addFile, addition, true))
620 {
621 LOG("Error loading file %s\n", (addDir+addFile).c_str());
622 return 1;
623 }
624
625 // for each language
626 for (uint l=0; l<Languages.size(); ++l)
627 {
628 LOG("Diffing with language %s...\n", Languages[l].c_str());
629
630 if (l != 0)
631 {
632 addition.clear();
633
634 std::string addFile = baseName + Languages[0] + ".uxt";
635 if (!loadStringFile(transDir+addFile, addition, true))
636 {
637 LOG("Error loading file %s\n", (transDir+addFile).c_str());
638 return 1;
639 }
640 }
641
642 vector<TStringInfo> reference;
643 // load the reference file
644 std::string refFile = baseName + Languages[l] + ".uxt";
645 if (!loadStringFile(transDir+refFile, reference, false))
646 {
647 LOG("Error loading file %s\n", (transDir+refFile).c_str());
648 return 1;
649 }
650
651 // load any not merged diff file
652 if (!mergeStringDiff(reference, Languages[l], baseName, ".uxt", false))
653 {
654 LOG("Error will mergin diff file(s)\n");
655 return 1;
656 }
657
658 vector<TStringInfo> diff;
659 makeStringDiff(addition, reference, diff);
660
661 if (diff.empty())
662 {
663 LOG("No difference for %s.\n", Languages[l].c_str());
664 }
665 else
666 {
667 LOG("Writing difference file for %s.\n", Languages[l].c_str());
668 // build the diff file for each language.
669 ucstring str = prepareStringFile(diff, false);
670
671 // add the tag for non translation
672 str += nl + ucstring ("// REMOVE THE FOLOWING LINE WHEN TRANSLATION IS DONE") + nl + ucstring("// DIFF NOT TRANSLATED") + nl;
673
674 std::string diffName = diffDir + baseName + Languages[l] + "_diff_" + diffVersion + ".uxt";
675 CI18N::writeTextFile(diffName, str);
676
677 }
678 }
679
680 return 0;
681 }
682
683 /*
684 Remove the OLD VALUE from a file.
685 */
686 void cleanComment(const std::string & filename)
687 {
688 ucstring text;
689 uint nbOldValue=0;
690
691 CI18N::readTextFile(filename, text, false, false, CI18N::LINE_FMT_LF);
692
693 ucstring newText;
694 ucstring::size_type last = 0;
695 while ( last != ucstring::npos)
696 {
697 ucstring::size_type commentBegin = text.find(ucstring("/* OLD VALUE :"), last);
698 if (commentBegin == ucstring::npos)
699 {
700 newText += text.substr(last);
701 last = ucstring::npos;
702 }
703 else
704 {
705 ucstring::size_type size = commentBegin - last;
706 ucstring toAdd = text.substr(last, size);
707 newText += toAdd;
708 ucstring::size_type commentEnd = text.find(ucstring("*/"), commentBegin);
709 if (commentEnd != ucstring::npos) { commentEnd += 2 + nl.size(); }
710 last = commentEnd;
711 ++nbOldValue;
712 }
713 }
714 text = newText;
715 newText.clear();
716 last = 0;
717 while ( last != ucstring::npos)
718 {
719 ucstring::size_type commentBegin = text.find(ucstring("//"), last);
720 if (commentBegin == ucstring::npos)
721 {
722 newText += text.substr(last);
723 last = ucstring::npos;
724 }
725 else
726 {
727 ucstring::size_type size = commentBegin - last;
728 ucstring toAdd = text.substr(last, size);
729 newText += toAdd;
730 // case where // is the part of an url and isn't a comment
731 if (commentBegin > 4 && text.substr(commentBegin-1, 1) == ucstring(":"))
732 {
733 newText += "//";
734 last = commentBegin+2;
735 }
736 else
737 {
738 ucstring::size_type commentEnd = text.find(ucstring("\n"), commentBegin);
739 if (commentEnd != ucstring::npos)
740 {
741 commentEnd += 1;
742 ucstring comment = text.substr(commentBegin, commentEnd - commentBegin);
743 if (comment.find(ucstring("// HASH_VALUE")) != ucstring::npos
744 || comment.find(ucstring("// DIFF")) != ucstring::npos
745 || comment.find(ucstring("// REMOVE")) != ucstring::npos
746 || comment.find(ucstring("// INDEX")) != ucstring::npos
747 )
748 {
749 newText += comment;
750 }
751 }
752 last = commentEnd;
753 ++nbOldValue;
754 }
755 }
756 }
757 nlinfo("cleaning : %s, (%d comments deleted)...\n", filename.c_str(), nbOldValue);
758 CI18N::writeTextFile(filename , newText);
759 }
760
761 /*
762 REMOVE OLDVALUE: from a diff string file
763 */
764 int cleanStringDiff(int argc, char *argv[], const std::string &baseName)
765 {
766
767 LOG("Cleaning string diffs\n");
768
769 uint i,l;
770
771 for (l=0; l<Languages.size(); ++l)
772 {
773
774 vector<string> diffs;
775
776 getPathContentFiltered(diffDir + baseName + Languages[l] + "_diff_", ".uxt", diffs);
777 for (i=0; i<diffs.size(); ++i)
778 {
779 cleanComment(diffs[i]);
780 }
781 }
782 return 0;
783 }
784
785 int mergeStringDiff(int argc, char *argv[], const std::string &baseName)
786 {
787 LOG("Merging string diffs\n");
788
789 // for each language
790 uint l;
791 for (l=0; l<Languages.size(); ++l)
792 {
793 LOG("Merging for language %s...\n", Languages[l].c_str());
794 string filename = transDir + baseName + Languages[l] + ".uxt";
795 // load the translated file
796 vector<TStringInfo> translated;
797 if (!loadStringFile(filename, translated, false))
798 {
799 LOG("Error will loading file %s\n", filename.c_str());
800 return 1;
801 }
802
803 // append the translated diffs
804 mergeStringDiff(translated, Languages[l], baseName, ".uxt", true, true);
805
806 // prepare the addition string
807 ucstring str = prepareStringFile(translated, true);
808
809 {
810 // backup the original file
811 ucstring old;
812 CI18N::readTextFile(filename, old, true, false, CI18N::LINE_FMT_LF);
813 if (old != str)
814 CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
815 }
816
817 CI18N::writeTextFile(filename, str);
818 }
819
820 return 0;
821 }
822
823
824 /*
825 struct TFindPhrase : unary_function<TPhrase, bool>
826 {
827 string Identifier;
828 TFindPhrase (const string &identifier)
829 : Identifier(identifier)
830 {}
831 bool operator () (const TPhrase &phrase)
832 {
833 return phrase.Identifier == Identifier;
834 }
835 };
836 */
837
838
839 bool mergePhraseDiff2(vector<TPhrase> &phrases, const string &language, bool onlyTranslated, bool archiveDiff);
840 bool mergePhraseDiff(vector<TPhrase> &phrases, const string &language, bool onlyTranslated, bool archiveDiff = false)
841 {
842 vector<string> diffs;
843
844 getPathContentFiltered(diffDir+"phrase_"+language+"_diff_", ".txt", diffs);
845
846 for (uint i=0; i<diffs.size(); ++i)
847 {
848 if (onlyTranslated)
849 {
850 // Check if the diff is translated
851 ucstring text;
852 CI18N::readTextFile(diffs[i], text, false, false, CI18N::LINE_FMT_LF);
853 verifyVersion(text, 1);
854 if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
855 {
856 LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(diffs[i]).c_str());
857 for (i=i+1; i<diffs.size(); ++i)
858 LOG(" Merge of Diff file [%s] delayed.\n", CFile::getFilename(diffs[i]).c_str());
859 return true;
860 }
861 }
862
863 // we found a diff file for the addition file.
864 LOG("Adding %s diff as reference\n", diffs[i].c_str());
865 vector<TPhrase> diff;
866 if (!readPhraseFile1(diffs[i], diff, false))
867 return false;
868
869 for (uint j=0; j<diff.size(); ++j)
870 {
871 /* TDiffCommand command;
872 uint index;
873 uint index2;
874 */
875 TDiffInfo diffInfo;
876 if (!parseDiffCommandFromComment(diff[j].Comments, diffInfo))
877 {
878 if (j == diff.size()-1)
879 break;
880 else
881 {
882 nlwarning("Failed to parse diff command in '%s'", diff[j].Identifier.c_str());
883 return false;
884 }
885 }
886
887 switch(diffInfo.Command)
888 {
889 case diff_swap:
890 nlassertex(diffInfo.Index1 <= phrases.size(),
891 ("In SWAP, Index1 (%u) is not less than number of phrase (%u)", diffInfo.Index1, phrases.size()));
892 nlassertex(diffInfo.Index2 <= phrases.size(),
893 ("In SWAP Index2 (%u) is not less than number of phrase (%u)", diffInfo.Index2, phrases.size()));
894 swap(phrases[diffInfo.Index1], phrases[diffInfo.Index2]);
895 // remove the swap from the comments
896 diff[j].Comments = diff[j].Comments.substr(diff[j].Comments.find(nl)+2);
897 j--;
898 break;
899 case diff_add:
900 nlassertex(diffInfo.Index1 <= phrases.size(),
901 ("In ADD, Index1 (%u) is not less than number of phrase (%u)", diffInfo.Index1, phrases.size()));
902 phrases.insert(phrases.begin()+diffInfo.Index1, diff[j]);
903 break;
904 case diff_changed:
905 nlassertex(diffInfo.Index1 < phrases.size(),
906 ("In CHANGED, Index1 (%u) is not less than number of phrase (%u)", diffInfo.Index1, phrases.size()));
907 phrases[diffInfo.Index1] = diff[j];
908 break;
909 case diff_removed:
910 nlassertex(diffInfo.Index1 < phrases.size(),
911 ("In REMOVED, Index1 (%u) is not less than number of phrase (%u)", diffInfo.Index1, phrases.size()));
912 phrases.erase(phrases.begin()+diffInfo.Index1);
913 break;
914 case diff_keep:
915 nlassertex(diffInfo.Index1 < phrases.size(),
916 ("In KEEP, Index1 (%u) is not less than number of phrase (%u)", diffInfo.Index1, phrases.size()));
917 phrases[diffInfo.Index1].HashValue = diff[j].HashValue;
918 phrases[diffInfo.Index1].Comments = diff[j].Comments;
919 break;
920 default:
921 nlassert(false);
922 }
923 }
924
925 if (archiveDiff)
926 {
927 // move the diff file in the history dir
928 CFile::moveFile(historyDir+CFile::getFilename(diffs[i]), diffs[i]);
929 }
930 }
931
932 return true;
933 }
934
935
936
937 class CMakePhraseDiff : CMakeDiff<TPhrase, TPhraseDiffContext>::IDiffCallback
938 {
939 public:
940 void run(const vector<TPhrase> &addition, vector<TPhrase> &reference, vector<TPhrase> &diff)
941 {
942 TPhraseDiffContext context(addition, reference, diff);
943
944 CMakeDiff<TPhrase, TPhraseDiffContext> differ;
945 differ.makeDiff(this, context);
946 }
947
948 void onEquivalent(uint addIndex, uint refIndex, TPhraseDiffContext &context)
949 {
950 // nothing to do
951 }
952 void onAdd(uint addIndex, uint refIndex, TPhraseDiffContext &context)
953 {
954 TPhrase phrase = context.Addition[addIndex];
955 char temp[1024];
956 sprintf(temp, "// DIFF ADD %u ", addIndex);
957 phrase.Comments = ucstring(temp) + nl + phrase.Comments;
958
959 nlinfo("Added %s at %u", phrase.Identifier.c_str(), addIndex);
960 context.Diff.push_back(phrase);
961 }
962 void onRemove(uint addIndex, uint refIndex, TPhraseDiffContext &context)
963 {
964 TPhrase phrase = context.Reference[refIndex];
965 char temp[1024];
966 sprintf(temp, "// DIFF REMOVED %u ", addIndex);
967 // NB : on vire les commentaires car il pourrai contenir des merdes..
968 phrase.Comments = ucstring(temp) + nl;
969 for (uint i=0; i<phrase.Clauses.size(); ++i)
970 phrase.Clauses[i].Comments.erase();
971
972 nlinfo("Removed %s at %u", phrase.Identifier.c_str(), addIndex);
973 context.Diff.push_back(phrase);
974 }
975 void onChanged(uint addIndex, uint refIndex, TPhraseDiffContext &context)
976 {
977 ucstring chg;
978 // check what is changed.
979 if (context.Addition[addIndex].Parameters != context.Reference[refIndex].Parameters)
980 chg += "// Parameter list changed." + nl;
981 if (context.Addition[addIndex].Clauses.size() != context.Reference[refIndex].Clauses.size())
982 chg += "// Clause list changed." + nl;
983 else
984 {
985 for (uint i=0; i<context.Addition[addIndex].Clauses.size(); ++i)
986 {
987 if (context.Addition[addIndex].Clauses[i].Identifier != context.Reference[refIndex].Clauses[i].Identifier)
988 chg += ucstring("// Clause ") + toString(i) + " : identifier changed." + nl;
989 else if (context.Addition[addIndex].Clauses[i].Conditions != context.Reference[refIndex].Clauses[i].Conditions)
990 chg += ucstring("// Clause ") + toString(i) + " : condition changed." + nl;
991 else if (context.Addition[addIndex].Clauses[i].Text != context.Reference[refIndex].Clauses[i].Text)
992 chg += ucstring("// Clause ") + toString(i) + " : text changed." + nl;
993 }
994 }
995
996 if (chg.empty())
997 {
998 chg = ucstring("// WARNING : Hash code changed ! check translation workflow.") + nl;
999 }
1000 nldebug("Changed detected : %s", chg.toString().c_str());
1001
1002 // changed element
1003 TPhrase phrase = context.Addition[addIndex];
1004 vector<TPhrase> tempV;
1005 tempV.push_back(context.Reference[refIndex]);
1006 ucstring tempT = preparePhraseFile(tempV, false);
1007 CI18N::removeCComment(tempT);
1008 phrase.Comments = ucstring("// DIFF CHANGED ") + toString(addIndex) + nl + phrase.Comments;
1009 phrase.Comments = phrase.Comments + ucstring("/* OLD VALUE : [" + nl) + tabLines(1, tempT) + nl + "] */" + nl;
1010 phrase.Comments = phrase.Comments + chg;
1011
1012 nlinfo("Changed %s at %u", phrase.Identifier.c_str(), addIndex);
1013 context.Diff.push_back(phrase);
1014 }
1015
1016 void onSwap(uint newIndex, uint refIndex, TPhraseDiffContext &context)
1017 {
1018 TPhrase phrase;
1019 char temp[1024];
1020 sprintf(temp, "// DIFF SWAP %u %u (swaping %s and %s)", newIndex, refIndex, context.Reference[newIndex].Identifier.c_str(), context.Reference[refIndex].Identifier.c_str());
1021
1022 nldebug("Swap for %u %u", newIndex, refIndex);
1023 phrase.Comments = ucstring(temp) + nl;
1024 context.Diff.push_back(phrase);
1025 }
1026
1027 };
1028
1029
1030
1031 int makePhraseDiff(int argc, char *argv[])
1032 {
1033 // Generate the diff file from phrase_<lang>.txt compared to the same file in translated.
1034 // The diff is generated only from the reference language for and all the languages
1035
1036 LOG("Generating phrase diffs\nLoading the working file for language %s\n", Languages[0].c_str());
1037
1038
1039 vector<TPhrase> addition;
1040
1041 // read addition
1042 if (!readPhraseFile(addDir+"phrase_"+Languages[0]+".txt", addition, true))
1043 {
1044 LOG("Error will loading file %s", (addDir+"phrase_"+Languages[0]+".txt").c_str());
1045 return 1;
1046 }
1047
1048 for (uint l =0; l<Languages.size(); ++l)
1049 {
1050 LOG("Diffing with language %s...\n", Languages[l].c_str());
1051 if (l == 1)
1052 {
1053 addition.clear();
1054 // read the language 0 translated version as addition for other language
1055 if (!readPhraseFile(transDir+"phrase_"+Languages[0]+".txt", addition, true))
1056 {
1057 LOG("Error will loading file %s", (addDir+"phrase_"+Languages[0]+".txt").c_str());
1058 return 1;
1059 }
1060 }
1061 vector<TPhrase> reference;
1062 // read the reference file
1063 if (!readPhraseFile(transDir+"phrase_"+Languages[l]+".txt", reference, false))
1064 {
1065 LOG("Error will loading file %s", (transDir+"phrase_"+Languages[l]+".txt").c_str());
1066 return 1;
1067 }
1068
1069 if (!mergePhraseDiff(reference, Languages[l], false))
1070 {
1071 LOG("Error will merging phrase diff for language %s\n", Languages[l].c_str());
1072 return 1;
1073 }
1074
1075 // compare the reference an addition file, remove any equivalent strings.
1076 uint addCount=0, refCount=0;
1077 vector<TPhrase> diff;
1078
1079 CMakePhraseDiff differ;
1080 differ.run(addition, reference, diff);
1081
1082 if (diff.empty())
1083 {
1084 LOG("No difference for language %s\n", Languages[l].c_str());
1085 }
1086 else
1087 {
1088 LOG("Writing difference file for language %s\n", Languages[l].c_str());
1089 ucstring text;
1090 text += "// DIFF_VERSION 1\n";
1091 text += preparePhraseFile(diff, false);
1092 // add the tag for non translation
1093 text += nl + ucstring ("// REMOVE THE FOLOWING LINE WHEN TRANSLATION IS DONE") + nl + ucstring("// DIFF NOT TRANSLATED") + nl;
1094 CI18N::writeTextFile(diffDir+"phrase_"+Languages[l]+"_diff_"+diffVersion+".txt", text);
1095 }
1096 }
1097
1098 return 0;
1099 }
1100
1101
1102 /*
1103 REMOVE OLDVALUE: from a diff clause file
1104 */
1105 int cleanPhraseDiff(int argc, char *argv[])
1106 {
1107
1108 LOG("Cleaning phrase diffs\n");
1109
1110 uint i,l;
1111
1112 for (l=0; l<Languages.size(); ++l)
1113 {
1114
1115 vector<string> diffs;
1116
1117 getPathContentFiltered(diffDir+"phrase_"+Languages[l]+"_diff_", ".txt", diffs);
1118 for (i=0; i<diffs.size(); ++i)
1119 {
1120 cleanComment(diffs[i]);
1121
1122 }
1123 }
1124 return 0;
1125 }
1126
1127
1128 int mergePhraseDiff(int argc, char *argv[], int version)
1129 {
1130 // merge all the phrase diff back into there repective translated phrase.
1131 uint l;
1132
1133 LOG("Merging phrase diffs\n");
1134
1135 for (l=0; l<Languages.size(); ++l)
1136 {
1137 LOG("Merging for language %s...\n", Languages[l].c_str());
1138 std::string basename("phrase_"+Languages[l]);
1139 string filename = transDir+basename+".txt";
1140 // build the addition diff
1141 vector<TPhrase> reference;
1142
1143 ucstring doc;
1144
1145
1146 if (!readPhraseFile(transDir+basename+".txt", reference, false))
1147 {
1148 LOG("Error will loading file %s", (transDir+basename+".txt").c_str());
1149 return 1;
1150 }
1151
1152
1153 switch(version)
1154 {
1155 case 1:
1156 if (!mergePhraseDiff(reference, Languages[l], true, true))
1157 {
1158 LOG("Error will merging phrase diff");
1159 return 1;
1160 }
1161 break;
1162
1163 case 2:
1164
1165 if (!mergePhraseDiff2(reference, Languages[l], true, true))
1166 {
1167 LOG("Error will merging phrase diff");
1168 return 1;
1169 }
1170 break;
1171
1172 default:
1173 nlassert(0);
1174
1175 }
1176
1177 ucstring str = preparePhraseFile(reference, true);
1178
1179 {
1180 // backup the original file
1181 ucstring old;
1182 CI18N::readTextFile(filename, old, true, false, CI18N::LINE_FMT_LF);
1183 if (old != str)
1184 CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
1185 }
1186
1187 CI18N::writeTextFile(transDir+basename+".txt", str);
1188
1189 }
1190
1191 return 0;
1192 }
1193
1194
1195 int makeClauseDiff(int argc, char *argv[])
1196 {
1197 // this will generate diff from 'addition' directory
1198 // for all the clause_<lang>.txt file
1199 // with the same file in the 'translated' directory.
1200
1201 // NB : we use standard C file access because there are mutiple file with the same name in different place.
1202
1203 LOG("Generating clause diffs\n");
1204
1205 uint i,l;
1206
1207 for (l=0; l<Languages.size(); ++l)
1208 {
1209 LOG("Diffing with language %s...\n", Languages[l].c_str());
1210 std::string basename("clause_"+Languages[l]);
1211 vector<TStringInfo> addition;
1212 vector<TStringInfo> reference;
1213 vector<TPhrase> phrases;
1214 std::vector<std::string> warnings;
1215
1216 // load the reference file
1217 std::string refFile(basename+".txt");
1218 if (!loadStringFile(transDir+refFile, reference, false))
1219 {
1220 LOG("Error will loading file %s", (transDir+refFile).c_str());
1221 return 1;
1222 }
1223
1224 // load the addition file
1225 std::string addFile("phrase_"+Languages[l]+".txt");
1226 if (!readPhraseFile(transDir+addFile, phrases, true))
1227 {
1228 LOG("Error will loading file %s", (transDir+addFile).c_str());
1229 return 1;
1230 }
1231
1232 // extract all the clauses from the phrases file
1233 vector<TPhrase>::iterator first(phrases.begin()), last(phrases.end());
1234 for (; first != last; ++first)
1235 {
1236 TPhrase &p = *first;
1237 for (i=0; i<p.Clauses.size(); ++i)
1238 {
1239 TStringInfo si;
1240 si.Comments = p.Clauses[i].Comments;
1241 si.Identifier = p.Clauses[i].Identifier;
1242 si.Text = p.Clauses[i].Text;
1243 si.HashValue = CI18N::makeHash(si.Text);
1244
1245
1246 if (!si.Identifier.empty())
1247 {
1248 vector<TStringInfo>::const_iterator first2 = addition.begin();
1249 vector<TStringInfo>::const_iterator last2 = addition.end();
1250 for ( ;first2!=last2 && first2->Identifier != si.Identifier; ++first2) {}
1251 bool isAllreadyThere = first2 != last2;
1252 if (isAllreadyThere)
1253 {
1254 warnings.push_back("The clause " +si.Identifier +" in the phrase " + p.Identifier +" exists more than once.");
1255 }
1256 else
1257 {
1258 addition.push_back(si);
1259 }
1260 }
1261 }
1262 }
1263
1264 if (!warnings.empty())
1265 {
1266 std::vector<std::string>::const_iterator first = warnings.begin();
1267 std::vector<std::string>::const_iterator last = warnings.end();
1268 for (;first != last; ++first) { nlwarning("%s", first->c_str()); }
1269 return -1;
1270 }
1271 mergeStringDiff(reference, Languages[l], "clause_", ".txt", false);
1272
1273 vector<TStringInfo> diff;
1274
1275 makeStringDiff(addition, reference, diff);
1276
1277 if (diff.empty())
1278 {
1279 LOG("No difference for language %s\n", Languages[l].c_str());
1280 }
1281 else
1282 {
1283 LOG("Writing difference file for %s.\n", Languages[l].c_str());
1284 // build the diff file for each language.
1285 ucstring str = prepareStringFile(diff, false);
1286
1287 // add the tag for non translation
1288 str += nl + ucstring ("// REMOVE THE FOLOWING LINE WHEN TRANSLATION IS DONE") + nl + ucstring("// DIFF NOT TRANSLATED") + nl;
1289
1290 std::string diffName(diffDir+"clause_"+Languages[l]+"_diff_"+diffVersion+".txt");
1291 CI18N::writeTextFile(diffName, str);
1292 }
1293 }
1294
1295 return 0;
1296 }
1297
1298
1299 /*
1300 REMOVE OLDVALUE: from a diff clause file
1301 */
1302 int cleanClauseDiff(int argc, char *argv[])
1303 {
1304
1305 LOG("Cleaning clause diffs\n");
1306
1307 uint i,l;
1308
1309 for (l=0; l<Languages.size(); ++l)
1310 {
1311
1312 std::string basename("clause_"+Languages[l]);
1313
1314 vector<string> diffs;
1315
1316 getPathContentFiltered(diffDir+"clause_"+Languages[l]+"_diff_", ".txt", diffs);
1317 for (i=0; i<diffs.size(); ++i)
1318 {
1319 cleanComment(diffs[i]);
1320 }
1321 }
1322 return 0;
1323 }
1324
1325 int mergeClauseDiff(int argc, char *argv[])
1326 {
1327 LOG("Merging clause diffs\n");
1328 // for each language
1329 uint l;
1330 for (l=0; l<Languages.size(); ++l)
1331 {
1332 LOG("Merging for language %s...\n", Languages[l].c_str());
1333 string filename = transDir+"clause_"+Languages[l]+".txt";
1334 // load the translated file
1335 vector<TStringInfo> translated;
1336 if (!loadStringFile(filename, translated, false))
1337 {
1338 LOG("Error will loading file %s", filename.c_str());
1339 return 1;
1340 }
1341
1342 // append the translated diffs
1343 mergeStringDiff(translated, Languages[l], "clause_", ".txt", true, true);
1344
1345 // prepare the addition string
1346 ucstring str = prepareStringFile(translated, true);
1347
1348 {
1349 // backup the original file
1350 ucstring old;
1351 CI18N::readTextFile(filename, old, true, false, CI18N::LINE_FMT_LF);
1352 if (old != str)
1353 CFile::moveFile(historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename), filename);
1354 }
1355
1356 CI18N::writeTextFile(filename, str);
1357 }
1358
1359 return 0;
1360
1361 return 0;
1362 }
1363
1364 bool mergeWorksheetDiff(const std::string filename, TWorksheet &sheet, bool onlyTranslated, bool archiveDiff)
1365 {
1366 std::string fn(CFile::getFilenameWithoutExtension(filename)), ext(CFile::getExtension(filename));
1367 vector<string> fileList;
1368 getPathContentFiltered(diffDir+fn+"_diff_", ext, fileList);
1369
1370 uint i;
1371 for (i=0; i<fileList.size(); ++i)
1372 {
1373 if (onlyTranslated)
1374 {
1375 ucstring text;
1376 CI18N::readTextFile(fileList[i], text, false, false, CI18N::LINE_FMT_LF);
1377 if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
1378 {
1379 LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(fileList[i]).c_str());
1380 for (i=i+1; i<fileList.size(); ++i)
1381 LOG(" Merge of Diff file [%s] delayed.\n", CFile::getFilename(fileList[i]).c_str());
1382 return true;
1383 }
1384 }
1385
1386 TWorksheet diff;
1387 if (!loadExcelSheet(fileList[i], diff, false))
1388 return false;
1389 makeHashCode(diff, false);
1390
1391 uint cmdCol = 0;
1392 if (!diff.findCol(ucstring("DIFF_CMD"), cmdCol))
1393 {
1394 LOG("Can't find DIFF_CMD column in %s ! Invalid diff file.\n", CFile::getFilename(fileList[i]).c_str());
1395 return false;
1396 }
1397
1398 uint hashCol;
1399 if (!diff.findCol(ucstring("*HASH_VALUE"), hashCol))
1400 hashCol = ~0;
1401
1402 // we found a diff file for the addition file.
1403 LOG("Adding %s diff as reference\n", fileList[i].c_str());
1404
1405 for (uint j=1; j<diff.Data.size(); ++j)
1406 {
1407 TDiffInfo diffInfo;
1408 if (!parseDiffCommandFromComment(diff.getData(j, cmdCol), diffInfo))
1409 {
1410 if (diff.getData(j, cmdCol).find(ucstring("REMOVE THE FOLOWING TWO LINE WHEN TRANSLATION IS DONE")) == ucstring::npos
1411 && diff.getData(j, cmdCol).find(ucstring("DIFF NOT TRANSLATED")) == ucstring::npos)
1412 return false;
1413 else
1414 continue;
1415 }
1416
1417 switch(diffInfo.Command)
1418 {
1419 case diff_add:
1420 {
1421 nlassertex(diffInfo.Index1 <= sheet.Data.size(),
1422 ("ADD cmd in diff file reference row %u, but worksheet only contains %u entries",
1423 diffInfo.Index1, sheet.Data.size()));
1424 TWorksheet::TRow row(sheet.ColCount);
1425 sheet.Data.insert(sheet.Data.begin()+diffInfo.Index1, row);
1426 for (uint k=0; k<diff.ColCount; ++k)
1427 {
1428 if (k != cmdCol)
1429 sheet.setData(diffInfo.Index1, diff.Data[0][k], diff.Data[j][k]);
1430 }
1431 }
1432 break;
1433 case diff_changed:
1434 {
1435 nlassertex(diffInfo.Index1 <= sheet.Data.size(),
1436 ("CHANGED cmd in diff file reference row %u, but worksheet only contains %u entries",
1437 diffInfo.Index1, sheet.Data.size()));
1438 for (uint k=0; k<diff.ColCount; ++k)
1439 {
1440 if (k != cmdCol)
1441 sheet.setData(diffInfo.Index1, diff.Data[0][k], diff.Data[j][k]);
1442 }
1443 }
1444 break;
1445 case diff_removed:
1446 nlassertex(diffInfo.Index1 < sheet.Data.size(),
1447 ("REMOVE cmd in diff file reference row %u, but worksheet only contains %u entries",
1448 diffInfo.Index1, sheet.Data.size()));
1449 // nlassertex(diffInfo.Index1 > 0);
1450 sheet.Data.erase(sheet.Data.begin() + diffInfo.Index1);
1451 break;
1452 case diff_swap:
1453 nlassertex(diffInfo.Index1 < sheet.Data.size(),
1454 ("SWAP cmd in diff file, first index reference row %u, but worksheet only contains %u entries",
1455 diffInfo.Index1, sheet.Data.size()));
1456 // nlassertex(diffInfo.Index1 > 0);
1457 nlassertex(diffInfo.Index2 < sheet.Data.size(),
1458 ("SWAP cmd in diff file, second index reference row %u, but worksheet only contains %u entries",
1459 diffInfo.Index1, sheet.Data.size()));
1460 // nlassertex(diffInfo.Index2 > 0);
1461 swap(sheet[diffInfo.Index1], sheet[diffInfo.Index2]);
1462 break;
1463 case diff_keep:
1464 {
1465 nlassertex(diffInfo.Index1 <= sheet.Data.size(),
1466 ("KEEP cmd in diff file reference row %u, but worksheet only contains %u entries",
1467 diffInfo.Index1, sheet.Data.size()));
1468 if (hashCol == ~0)
1469 {
1470 nlerror("Hash column not available, keep not implemented");
1471 }
1472 else
1473 {
1474 sheet.setData(diffInfo.Index1, diff.Data[0][hashCol], diff.Data[j][hashCol]);
1475 }
1476 }
1477 break;
1478 default:
1479 nlassert(false);
1480 }
1481
1482 }
1483
1484 if (archiveDiff)
1485 {
1486 // move the diff file in the history dir
1487 CFile::moveFile(historyDir+CFile::getFilename(fileList[i]), fileList[i]);
1488 }
1489 }
1490
1491 return true;
1492 }
1493
1494
1495
1496 bool mergeSheetDiff(const string &type, TWorksheet &sheet, const string &language, bool onlyTranslated, bool archiveDiff)
1497 {
1498 return mergeWorksheetDiff(type+"_words_"+language+".txt", sheet, onlyTranslated, archiveDiff);
1499 }
1500
1501
1502 class CMakeWordsDiff : public TWorkSheetDiff::IDiffCallback
1503 {
1504 public:
1505 void run(const TWorksheet &addition, TWorksheet &reference, TWorksheet &diff)
1506 {
1507 TWordsDiffContext context(addition, reference, diff);
1508
1509 TWorkSheetDiff differ;
1510 differ.makeDiff(this, context, true);
1511 }
1512
1513 void onEquivalent(uint addIndex, uint refIndex, TWordsDiffContext &context)
1514 {
1515 // nothing to do
1516 }
1517 void onAdd(uint addIndex, uint refIndex, TWordsDiffContext &context)
1518 {
1519 TWorksheet::TRow row(context.Reference.ColCount+1);
1520 for (uint j=0; j<context.Addition.ColCount; ++j)
1521 {
1522 uint colIndex = 0;
1523 if (context.Reference.findCol(context.Addition.Data[0][j], colIndex))
1524 {
1525 row[colIndex+1] = context.Addition.Data[addIndex][j];
1526 }
1527 }
1528 char temp[1024];
1529 sprintf(temp, "DIFF ADD %u ", addIndex);
1530 row[0] = ucstring(temp);
1531
1532 nlinfo("Added %s at %u", row[2].toString().c_str(), addIndex);
1533 context.Diff.insertRow((uint)context.Diff.Data.size(), row);
1534 }
1535 void onRemove(uint addIndex, uint refIndex, TWordsDiffContext &context)
1536 {
1537 TWorksheet::TRow row(context.Reference.ColCount+1);
1538 for (uint j=0; j<context.Reference.ColCount; ++j)
1539 {
1540 uint colIndex = 0;
1541 if (context.Reference.findCol(context.Reference.Data[0][j], colIndex))
1542 {
1543 row[colIndex+1] = context.Reference.Data[refIndex][j];
1544 }
1545 }
1546 char temp[1024];
1547 sprintf(temp, "DIFF REMOVED %u ", refIndex);
1548 row[0] = ucstring(temp);
1549
1550 nlinfo("Removed %s at %u", row[2].toString().c_str(), refIndex);
1551 context.Diff.insertRow((uint)context.Diff.Data.size(), row);
1552 }
1553 void onChanged(uint addIndex, uint refIndex, TWordsDiffContext &context)
1554 {
1555 TWorksheet::TRow row; //(context.Reference.ColCount+1);
1556 // copy the old content (this fill data in column that don't exist in addition worksheet)
1557 row = context.Reference.Data[refIndex];
1558 row.insert(row.begin(), ucstring());
1559
1560 // changed element
1561 for (uint j=0; j<context.Addition.ColCount; ++j)
1562 {
1563 uint colIndex = 0;
1564 if (context.Reference.findCol(context.Addition.Data[0][j], colIndex))
1565 {
1566 row[colIndex+1] = context.Addition.Data[addIndex][j];
1567 }
1568 }
1569
1570 char temp[1024];
1571 sprintf(temp, "DIFF CHANGED %u ", addIndex);
1572 row[0] = temp;
1573
1574 nlinfo("Changed %s at %u", row[2].toString().c_str(), addIndex);
1575 context.Diff.insertRow((uint)context.Diff.Data.size(), row);
1576 }
1577
1578 void onSwap(uint newIndex, uint refIndex, TWordsDiffContext &context)
1579 {
1580 TWorksheet::TRow row(context.Reference.ColCount+1);
1581 // swap
1582 char temp[1024];
1583 sprintf(temp, "DIFF SWAP %u %u", newIndex, refIndex);
1584 row[0] = temp;
1585
1586 nlinfo("Swap %u with %u", newIndex, refIndex);
1587 context.Diff.insertRow((uint)context.Diff.Data.size(), row);
1588 }
1589
1590 };
1591
1592 /*
1593 REMOVE OLDVALUE: from a diff words file
1594 */
1595 int cleanWordsDiff(int argc, char *argv[])
1596 {
1597
1598 LOG("Cleaning words diffs\n");
1599
1600 uint i,l;
1601
1602 for (l=0; l<Languages.size(); ++l)
1603 {
1604
1605 vector<string> diffs;
1606
1607 getPathContentFiltered(diffDir+"clause_"+Languages[l]+"_diff_", ".txt", diffs);
1608 for (i=0; i<diffs.size(); ++i)
1609 {
1610 cleanComment(diffs[i]);
1611 }
1612 }
1613
1614 return 0;
1615 }
1616
1617 int makeWorksheetDiff(int argc, char *argv[], const std::string &additionFilename, const std::string &referenceFilename, bool firstLanguage)
1618 {
1619 /* if (argc != 3)
1620 {
1621 LOG("ERROR : makeWorksheetDiff need a worksheet file in parameter !");
1622 return 1;
1623 }
1624 std::string filename = argv[2];
1625 */
1626 LOG("Loading working for %s...\n", referenceFilename.c_str());
1627
1628 // loads the working file
1629 TWorksheet addition;
1630 if (firstLanguage)
1631 {
1632 if (!loadExcelSheet(addDir + additionFilename, addition))
1633 return false;
1634 }
1635 else
1636 {
1637 if (!loadExcelSheet(transDir + additionFilename, addition))
1638 return false;
1639 }
1640
1641 makeHashCode(addition, true);
1642
1643 TWorksheet reference;
1644 if (CFile::fileExists(transDir+referenceFilename))
1645 {
1646 // load the sheet
1647 if (!loadExcelSheet(transDir+referenceFilename, reference))
1648 {
1649 LOG("Error reading worksheet file '%s'", (transDir+referenceFilename).c_str());
1650 return false;
1651 }
1652 }
1653 if (!CFile::fileExists(transDir+referenceFilename))
1654 {
1655 // init the reference column with addition column
1656 TWorksheet::TRow row(addition.ColCount);
1657 for (uint j=0; j<addition.ColCount; ++j)
1658 {
1659 nldebug("Adding column %s into reference sheet", addition.Data[0][j].toString().c_str());
1660 row[j] = addition.Data[0][j];
1661 reference.insertColumn(0);
1662 }
1663 reference.insertRow(0, row);
1664 }
1665 makeHashCode(reference, false);
1666
1667 mergeWorksheetDiff(referenceFilename, reference, false, false);
1668 // mergeSheetDiff(type, reference, Languages[l], false, false);
1669
1670 // generate the diff
1671 TWorksheet diff;
1672 TWorksheet::TRow row(reference.ColCount+1);
1673 // create the needed column.
1674 row[0] = ucstring("DIFF_CMD");
1675 diff.insertColumn(0);
1676 for (uint j=0; j<reference.ColCount; ++j)
1677 {
1678 row[j+1] = reference.Data[0][j];
1679 diff.insertColumn(j+1);
1680 }
1681 diff.insertRow(0, row);
1682
1683 CMakeWordsDiff differ;
1684 differ.run(addition, reference, diff);
1685
1686
1687 // write the diff file
1688 if (diff.Data.size() <= 1)
1689 {
1690 LOG("No difference for '%s'.\n", referenceFilename.c_str());
1691 }
1692 else
1693 {
1694 LOG("Writing difference file for %s.\n", referenceFilename.c_str());
1695 // build the diff file for each language.
1696 ucstring str = prepareExcelSheet(diff);
1697
1698 // add the tag for non translation
1699 str += ucstring ("REMOVE THE FOLOWING TWO LINE WHEN TRANSLATION IS DONE") + nl + ucstring("DIFF NOT TRANSLATED") + nl;
1700
1701 string fn(CFile::getFilenameWithoutExtension(referenceFilename)), ext(CFile::getExtension(referenceFilename));
1702 std::string diffName(diffDir+fn+"_diff_"+diffVersion+"."+ext);
1703 CI18N::writeTextFile(diffName, str);
1704
1705 }
1706
1707 return 0;
1708 }
1709
1710 int mergeWorksheetDiff(int argc, char *argv[], const std::string &filename, const string &additionFile)
1711 {
1712 /* if (argc != 3)
1713 {
1714 LOG("ERROR : mergeWorksheetDiff need a worksheet file in parameter !");
1715 return 1;
1716 }
1717 std::string filename = argv[2];
1718 */
1719 LOG("Merging for file '%s'...\n", filename.c_str());
1720 // string filename = transDir+types[t]+"_words_"+Languages[l]+".txt";
1721 // load the translated file
1722 TWorksheet translated;
1723 if (!CFile::fileExists(transDir+filename) || !loadExcelSheet(transDir+filename, translated))
1724 {
1725 // there is no translated file yet, build one from the working file.
1726 ucstring str;
1727 string addfn = addDir+additionFile;
1728 CI18N::readTextFile(addfn, str, false, false, CI18N::LINE_FMT_LF);
1729 str = str.substr(0, str.find(nl)+2);
1730 CI18N::writeTextFile(transDir+filename, str);
1731 // reread the file.
1732 bool res = loadExcelSheet(transDir+filename, translated);
1733 nlassert(res);
1734
1735 }
1736 makeHashCode(translated, false);
1737
1738 // append the translated diffs
1739 mergeWorksheetDiff(filename, translated, true, true);
1740 // mergeSheetDiff(types[t], translated, Languages[l], true, true);
1741
1742 // prepare the addition string
1743 ucstring str = prepareExcelSheet(translated);
1744
1745 {
1746 // backup the original file
1747 ucstring old;
1748 CI18N::readTextFile(transDir+filename, old, true, false, CI18N::LINE_FMT_LF);
1749 if (old != str)
1750 {
1751 string fn(CFile::getFilenameWithoutExtension(filename)), ext(CFile::getExtension(filename));
1752 CFile::moveFile((historyDir+fn+"_"+diffVersion+"."+ext).c_str(), (transDir+filename).c_str());
1753 }
1754 }
1755
1756 if (translated.size() > 0)
1757 CI18N::writeTextFile(transDir+filename, str);
1758
1759 return 0;
1760 }
1761
1762
1763 int makeWordsDiff(int argc, char *argv[])
1764 {
1765 vector<string> fileList;
1766 CPath::getPathContent(addDir, false, false, true, fileList);
1767
1768 // filter in words file only
1769 uint i;
1770 for (i=0; i<fileList.size(); ++i)
1771 {
1772 if (fileList[i].find("_words_"+Languages[0]+".txt") == string::npos || fileList[i].find(".#") != string::npos )
1773 {
1774 fileList.erase(fileList.begin()+i);
1775 --i;
1776 }
1777 }
1778
1779 int ret = 0;
1780
1781 // for each word file
1782 for (uint i=0; i<fileList.size(); ++i)
1783 {
1784 string type;
1785 type = CFile::getFilename(fileList[i]);
1786 type = type.substr(0, type.find("_") );
1787
1788 for (uint l=0; l<Languages.size(); ++l)
1789 {
1790 LOG("Diffing for language %s, type %s...\n", Languages[l].c_str(), type.c_str());
1791
1792 if (l == 0)
1793 ret += makeWorksheetDiff(argc, argv, CFile::getFilename(fileList[i]), CFile::getFilename(fileList[i]), true);
1794 else
1795 ret += makeWorksheetDiff(argc, argv, CFile::getFilename(fileList[i]), type+"_words_"+Languages[l]+".txt", false);
1796 }
1797 }
1798
1799 return ret;
1800
1801 }
1802
1803 int mergeWordsDiff(int argc, char *argv[])
1804 {
1805 LOG("Merging words diffs\n");
1806
1807 int ret = 0;
1808
1809 vector<string> fileList;
1810 CPath::getPathContent(addDir, false, false, true, fileList);
1811
1812 // filter in words file only
1813 for (uint i=0; i<fileList.size(); ++i)
1814 {
1815 if (fileList[i].find("_words_"+Languages[0]+".txt") == string::npos)
1816 {
1817 fileList.erase(fileList.begin()+i);
1818 --i;
1819 }
1820 }
1821
1822 // for each language
1823 for (uint l=0; l<Languages.size(); ++l)
1824 {
1825 // for each file
1826 for (uint i=0; i<fileList.size(); ++i)
1827 {
1828 string type;
1829 type = CFile::getFilename(fileList[i]);
1830 type = type.substr(0, type.find("_") );
1831
1832 ret += mergeWorksheetDiff(argc, argv, type+"_words_"+Languages[l]+".txt", CFile::getFilename(fileList[i]));
1833 }
1834 }
1835
1836 return ret;
1837
1838 }
1839
1840
1841
1842 /// temporary code
1843 struct TMissionInfo
1844 {
1845 ucstring Info;
1846 string Title;
1847 string Detail;
1848 string EndDetail;
1849 string Step0Desc;
1850 string Step0Prog;
1851 string Step0ProgDesc;
1852 };
1853
1854 struct TCompCondNum
1855 {
1856 bool operator() (const TClause &c1, const TClause &c2)
1857 {
1858 return count(c1.Conditions.begin(), c1.Conditions.end(), '&') > count(c2.Conditions.begin(), c2.Conditions.end(), '&');
1859 }
1860 };
1861
1862 int recupAround(int argc, char *argv[])
1863 {
1864 string clause1(diffDir+"clause_en_diff_3E896220.txt");
1865 string clause2(addDir+"clause_en_diff_3E7B4CE4 TRANSLATED.txt");
1866
1867 vector<TStringInfo> reference;
1868 loadStringFile(clause1, reference, true);
1869 vector<TStringInfo> around;
1870 loadStringFile(clause2, around, true, '[', ']', true);
1871
1872 vector<TStringInfo> result;
1873
1874 nlassert(reference.size() == around.size());
1875
1876 for (uint i=0; i<reference.size(); ++i)
1877 {
1878 TStringInfo si = reference[i];
1879 si.Text = around[i].Text2;
1880 si.Comments = around[i].Comments;
1881
1882 result.push_back(si);
1883 }
1884
1885 ucstring str = prepareStringFile(result, false);
1886
1887 CI18N::writeTextFile(addDir+"test_clause.txt", str);
1888
1889 return 0;
1890 }
1891
1892 //int mergeYannTaf();
1893 int addStringNumber();
1894
1895
1896 void cropLines(const std::string &filename, uint32 nbLines)
1897 {
1898 ucstring utext;
1899
1900 LOG("Cropping %u lines from file '%s'\n", nbLines, filename.c_str());
1901
1902 CI18N::readTextFile(filename, utext, false, false, CI18N::LINE_FMT_LF);
1903
1904 string text = utext.toUtf8();
1905
1906 vector<string> lines;
1907 explode(text, std::string("\n"), lines);
1908
1909 text.clear();
1910 if (lines.size() > nbLines)
1911 {
1912 for (uint i=0; i<lines.size()-nbLines; ++i)
1913 text += lines[i] + "\n";
1914 }
1915
1916 utext.fromUtf8(text);
1917
1918 CI18N::writeTextFile(filename, utext);
1919 }
1920
1921
1922
1923 int makeWork()
1924 {
1925 vector<string> files;
1926 uint i;
1927
1928 // move en.uxt file to wk.uxt
1929 CFile::moveFile((CPath::standardizePath(addDir)+"wk.uxt").c_str(), (CPath::standardizePath(addDir)+"en.uxt").c_str());
1930
1931 files.clear();
1932 CPath::getPathContent(addDir, true, false, true, files);
1933
1934 string strreplaced("_en.txt");
1935 string strtoreplace("_wk.txt");
1936
1937 for (i=0; i<files.size(); ++i)
1938 {
1939 if (testWildCard(CFile::getFilename(files[i]).c_str(), "*_en.txt"))
1940 {
1941 std::string filename = files[i];
1942 nlinfo("checking file '%s'", filename.c_str());
1943
1944 // change #include "*_en.txt" into #include "*_wk.txt"
1945 ucstring utext;
1946
1947 CI18N::readTextFile(filename, utext, false, false, CI18N::LINE_FMT_LF);
1948 string text = utext.toUtf8();
1949
1950 bool changedFile = false;
1951
1952 string::size_type p = 0;
1953 while ( (p=text.find("#include", p)) != string::npos)
1954 {
1955 string::size_type start = p, end;
1956 while (start < text.size() && text[start++] != '"')
1957 ;
1958 end = start;
1959 while (end < text.size() && text[end] != '"')
1960 ++end;
1961
1962 string includefilename = text.substr(start, end-start);
1963
1964 if (testWildCard(includefilename.c_str(), "*_en.txt"))
1965 {
1966 string originalfilename = includefilename;
1967 includefilename.replace(includefilename.size()-strreplaced.size(), strreplaced.size(), strtoreplace);
1968 text.replace(start, end-start, includefilename);
1969
1970 nlinfo("replaced '#include \"%s\"' into '#include \"%s\"'", originalfilename.c_str(), includefilename.c_str());
1971 changedFile = true;
1972 }
1973
1974 p = end;
1975 }
1976
1977 if (changedFile)
1978 {
1979 utext.fromUtf8(text);
1980 CI18N::writeTextFile(filename, utext);
1981 }
1982
1983 // change filename
1984 std::string movetofilename = filename;
1985 movetofilename.replace(movetofilename .size()-strreplaced.size(), strreplaced.size(), strtoreplace);
1986
1987 if (CFile::moveFile(movetofilename.c_str(), filename.c_str()))
1988 {
1989 nlinfo("moved file '%s' to '%s'", filename.c_str(), movetofilename.c_str());
1990 }
1991 else
1992 {
1993 nlwarning("FAILED to move file '%s' to '%s'", filename.c_str(), movetofilename.c_str());
1994 }
1995 }
1996 }
1997
1998 // move en.uxt file to wk.uxt
1999 CFile::moveFile((CPath::standardizePath(transDir)+"wk.uxt").c_str(), (CPath::standardizePath(transDir)+"en.uxt").c_str());
2000
2001 files.clear();
2002 CPath::getPathContent(transDir, true, false, true, files);
2003
2004 for (i=0; i<files.size(); ++i)
2005 {
2006 if (testWildCard(CFile::getFilename(files[i]).c_str(), "*_en.txt"))
2007 {
2008 std::string filename = files[i];
2009 nlinfo("checking file '%s'", filename.c_str());
2010
2011 // change filename
2012 std::string movetofilename = filename;
2013 movetofilename.replace(movetofilename .size()-strreplaced.size(), strreplaced.size(), strtoreplace);
2014
2015 nlinfo("moved file '%s' to '%s'", filename.c_str(), movetofilename.c_str());
2016
2017 CFile::moveFile(movetofilename.c_str(), filename.c_str());
2018 }
2019 }
2020
2021 return 0;
2022 }
2023
2024
2025 void preprocessTextFile(const std::string &filename,
2026 std::vector< std::pair<ucstring, std::string> > & outputResult);
2027
2028 void assertUniq(const vector<TPhrase>& reference)
2029 {
2030
2031 std::set< std::string > phraseIdentifier;
2032 std::set< std::string > clauseIdentifier;
2033 vector<TPhrase>::const_iterator first( reference.begin() );
2034 vector<TPhrase>::const_iterator last( reference.end() );
2035 for( ; first != last; ++first)
2036 {
2037 if ( phraseIdentifier.find(first->Identifier) != phraseIdentifier.end())
2038 {
2039 nlwarning("Phrase %s defined more than once.", first->Identifier.c_str());
2040 exit(-1);
2041 }
2042 else
2043 {
2044 phraseIdentifier.insert(first->Identifier);
2045 vector<TClause>::const_iterator first2( first->Clauses.begin() );
2046 vector<TClause>::const_iterator last2( first->Clauses.end() );
2047 for( ; first2 != last2; ++first2)
2048 {
2049 if (clauseIdentifier.find(first2->Identifier) != clauseIdentifier.end() )
2050 {
2051 nlwarning("Clause %s defined more than once.", first2->Identifier.c_str());
2052 exit(-1);
2053 }
2054 }
2055 }
2056 }
2057
2058 }
2059
2060
2061 void mergePhraseDiff2Impl(vector<TPhrase>& reference, const vector<TPhrase>& addition)
2062 {
2063 assertUniq(reference);
2064 assertUniq(addition);
2065
2066 typedef std::map<std::string, TPhrase> TMap;
2067
2068 TMap phrases;
2069
2070 {
2071 vector<TPhrase>::const_iterator first( reference.begin() );
2072 vector<TPhrase>::const_iterator last( reference.end() );
2073 for( ; first != last ; ++first )
2074 {
2075 std::string identifier = first->Identifier;
2076 phrases[identifier] = *first;
2077 }
2078 }
2079
2080 {
2081 vector<TPhrase>::const_iterator first( addition.begin() );
2082 vector<TPhrase>::const_iterator last( addition.end() );
2083 for( ; first != last ; ++first )
2084 {
2085 if ( first->Comments.find(ucstring("DIFF CHANGED")) != ucstring::npos)
2086 {
2087 nlassert( phrases.find(first->Identifier) != phrases.end() );
2088 phrases[first->Identifier] = *first;
2089 }
2090 else if ( first->Comments.find(ucstring("DIFF ADD")) != ucstring::npos)
2091 {
2092 nlassert( phrases.find(first->Identifier) == phrases.end() );
2093 phrases[first->Identifier] = *first;
2094 }
2095 else if ( first->Comments.find(ucstring("DIFF REMOVED")) != ucstring::npos)
2096 {
2097 nlassert( phrases.find(first->Identifier) != phrases.end() );
2098 phrases.erase( phrases.find(first->Identifier));
2099 }
2100 else if ( first->Comments.find(ucstring("DIFF KEEP")) != ucstring::npos)
2101 {
2102 nlassert( phrases.find(first->Identifier) != phrases.end() );
2103 phrases[first->Identifier].HashValue = first->HashValue;
2104 phrases[first->Identifier].Comments = first->Comments;
2105 }
2106 else
2107 {
2108 // nlassert(0 && "INVALID DIFF COMMAND");
2109 }
2110 }
2111 }
2112
2113 {
2114 reference.clear();
2115 reference.reserve(phrases.size());
2116 TMap::const_iterator first( phrases.begin() );
2117 TMap::const_iterator last( phrases.end() );
2118 for( ; first != last; ++first) { reference.push_back(first->second); }
2119 }
2120
2121 }
2122
2123 void removeHashValueComment(ucstring & comments)
2124 {
2125 ucstring::size_type first;
2126 ucstring::size_type last;
2127 first = comments.rfind(ucstring("// HASH_VALUE"));
2128 if (first != ucstring::npos)
2129 {
2130 last = comments.find(ucstring("\n"), first);
2131 if (last != ucstring::npos)
2132 {
2133 last += 1;
2134 ucstring tmp1 = comments.substr(0, first);
2135 ucstring tmp2 = last !=comments.size()
2136 ? comments.substr(last)
2137 : ucstring("");
2138 comments = tmp1 + tmp2;
2139 }
2140 else
2141 {
2142 comments = comments.substr(0, first);
2143 }
2144 }
2145 else
2146 {
2147 //comments = comments;
2148 }
2149
2150 }
2151
2152 bool updateClauseHashValue(const std::map<std::string, std::pair<uint64, uint64> >& validValues, const std::string & dirPath = "")
2153 {
2154
2155 for (uint l=0; l<Languages.size() ; ++l)
2156 {
2157
2158 std::string basename("clause_"+Languages[l]);
2159 vector<TStringInfo> clauses;
2160 std::string refFile(basename+".txt");
2161 if (!loadStringFile(transDir+refFile, clauses, false))
2162 {
2163 LOG("Error will loading file %s", (transDir+refFile).c_str());
2164 return false;
2165 }
2166
2167 bool changed = false;
2168 for ( uint i=0; i < clauses.size() ; ++i)
2169 {
2170 std::string Identifier = clauses[i].Identifier;
2171 if ( validValues.find(Identifier) != validValues.end())
2172 {
2173 if (!validValues.find(Identifier)->second.second
2174 || clauses[i].HashValue == validValues.find(Identifier)->second.second)
2175 {
2176 clauses[i].HashValue = validValues.find(Identifier)->second.first;
2177 removeHashValueComment(clauses[i].Comments);
2178 changed = true;
2179 }
2180 }
2181 }
2182
2183 if (!changed)
2184 {
2185 nlwarning("Clauses file don't need update for language %s\n", Languages[l].c_str());
2186 }
2187 else
2188 {
2189 nlinfo("Updating hashcode of clause file for %s.\n", Languages[l].c_str());
2190 // build the diff file for each language.
2191 ucstring str = prepareStringFile(clauses, false);
2192 std::string clauseName(dirPath+ transDir + basename +".txt");
2193 CFile::createDirectoryTree( CFile::getPath(clauseName) );
2194 CI18N::writeTextFile(clauseName, str);
2195 }
2196
2197 }
2198 return true;
2199 }
2200
2201 ucstring preparePhraseFile2(const vector<TPhrase> &phrases, bool removeDiffComments)
2202 {
2203 ucstring ret;
2204 vector<TPhrase>::const_iterator first(phrases.begin()), last(phrases.end());
2205 for (; first != last; ++first)
2206 {
2207 const TPhrase &p = *first;
2208
2209 if (removeDiffComments)
2210 {
2211 string comment = p.Comments.toString();
2212 vector<string> lines;
2213 explode(comment, std::string("\n"), lines, true);
2214
2215 uint i;
2216 for (i=0; i<lines.size(); ++i)
2217 {
2218 if (lines[i].find("// DIFF ") != string::npos)
2219 {
2220 lines.erase(lines.begin()+i);
2221 --i;
2222 }
2223 }
2224
2225 comment.erase();
2226 for (i=0; i<lines.size(); ++i)
2227 {
2228 comment += lines[i] + "\n";
2229 }
2230 p.Comments = ucstring(comment);
2231 }
2232 ret += p.Comments;
2233
2234 if (!p.Identifier.empty() || !p.Clauses.empty())
2235 {
2236 /*if (p.Comments.find(ucstring("// HASH_VALUE ")) == ucstring::npos)
2237 {
2238 // add the hash value.
2239 ret += ucstring("// HASH_VALUE ")+CI18N::hashToString(p.HashValue) + nl;
2240 }*/
2241 ret += p.Identifier + "("+p.Parameters + ")" + nl;
2242 ret += '{';
2243 ret += nl;
2244 for (uint i=0; i<p.Clauses.size(); ++i)
2245 {
2246 const TClause &c = p.Clauses[i];
2247 if (!c.Comments.empty())
2248 {
2249 ucstring comment = tabLines(1, c.Comments);
2250 if (comment[comment.size()-1] == ucchar(' ')) comment=comment.substr(0, comment.size() - 1);
2251 ret += comment; // + '\r'+'\n';
2252 }
2253 if (!c.Conditions.empty())
2254 {
2255 ucstring cond = tabLines(1, c.Conditions);
2256 ret += cond + nl;
2257 }
2258 ret += '\t';
2259
2260 ucstring text = CI18N::makeMarkedString('[', ']', c.Text);;
2261 ucstring text2;
2262 // add new line and tab after each \n tag
2263 ucstring::size_type pos;
2264 const ucstring nlTag("\\n");
2265 while ((pos = text.find(nlTag)) != ucstring::npos)
2266 {
2267 text2 += text.substr(0, pos+2) + nl;
2268 text = text.substr(pos+2);
2269 }
2270 text2 += text;//.substr(0, pos+2);
2271
2272 text.swap(text2);
2273
2274 text = tabLines(3, text);
2275 // remove begin tabs
2276 text = text.substr(3);
2277 ret += '\t' + (c.Identifier.empty()? "" : c.Identifier + ' ' )+ text + nl + nl;
2278 }
2279 ret += '}';
2280 }
2281 ret += nl + nl;
2282 }
2283
2284 return ret;
2285 }
2286
2287 bool updatePhraseHashValue(const std::map<std::string, std::pair<uint64, uint64> > & validValues, const std::string & dirPath = "")
2288 {
2289
2290 for (uint l=0; l<Languages.size() ; ++l)
2291 {
2292
2293 std::string basename("phrase_"+Languages[l]);
2294 vector<TPhrase> phrases;
2295 std::string refFile(basename+".txt");
2296 if (!readPhraseFile(transDir+refFile, phrases, false))
2297 {
2298 LOG("Error will loading file %s", (transDir+refFile).c_str());
2299 return false;
2300 }
2301
2302 bool changed = false;
2303 for ( uint i=0; i < phrases.size() ; ++i)
2304 {
2305 std::string Identifier = phrases[i].Identifier;
2306 if ( validValues.find(Identifier) != validValues.end())
2307 {
2308 if (!validValues.find(Identifier)->second.second || phrases[i].HashValue == validValues.find(Identifier)->second.second )
2309 {
2310
2311 phrases[i].HashValue = validValues.find(Identifier)->second.first;
2312 removeHashValueComment(phrases[i].Comments);
2313 changed = true;
2314 }
2315 }
2316 }
2317
2318 if (!changed)
2319 {
2320 nlinfo("Phrase file don't need update for language %s\n", Languages[l].c_str());
2321 }
2322 else
2323 {
2324 nlinfo("Updating hashcode of phrase file for %s.\n", Languages[l].c_str());
2325 // build the diff file for each language.
2326 ucstring str = preparePhraseFile(phrases, false);
2327 std::string pharseName(dirPath+ transDir + basename +".txt");
2328 CFile::createDirectoryTree( CFile::getPath(pharseName) );
2329 CI18N::writeTextFile(pharseName, str);
2330 }
2331
2332 }
2333 return true;
2334 }
2335
2336
2337 bool sortTransPhrase()
2338 {
2339
2340 for (uint l=0; l<Languages.size() ; ++l)
2341 {
2342
2343 std::string basename("phrase_"+Languages[l]);
2344 vector<TPhrase> phrases;
2345 vector<TPhrase> phrases2;
2346 std::map<std::string, TPhrase> phraseMap;
2347 std::string refFile(basename+".txt");
2348 if (!readPhraseFile(transDir+refFile, phrases, false))
2349 {
2350 LOG("Error will loading file %s", (transDir+refFile).c_str());
2351 return false;
2352 }
2353
2354 {
2355
2356 std::vector<TPhrase>::const_iterator first(phrases.begin());
2357 std::vector<TPhrase>::const_iterator last(phrases.end());
2358 for ( ; first != last; ++first)
2359 {
2360 phraseMap[first->Identifier] = *first;
2361 }
2362 }
2363 {
2364 std::map<std::string, TPhrase>::const_iterator first(phraseMap.begin());
2365 std::map<std::string, TPhrase>::const_iterator last(phraseMap.end());
2366 for ( ; first != last; ++first)
2367 {
2368 phrases2.push_back( first->second);
2369 }
2370 }
2371
2372 nlinfo("Updating hashcode of phrase file for %s.", Languages[l].c_str());
2373 // build the diff file for each language.
2374 ucstring str = preparePhraseFile(phrases2, false);
2375 std::string pharseName(transDir+refFile);
2376 CFile::createDirectoryTree( CFile::getPath(pharseName) );
2377 CI18N::writeTextFile(pharseName, str);
2378 }
2379
2380 return true;
2381 }
2382
2383
2384
2385 void patchWorkFile(vector<TPhrase> &updatedPhrase, const std::string & filename)
2386 {
2387 ucstring text;
2388 if ( updatedPhrase.empty() ) { return; }
2389 CI18N::readTextFile(filename, text, false, false, CI18N::LINE_FMT_LF);
2390 vector<TPhrase>::const_iterator first(updatedPhrase.begin());
2391 vector<TPhrase>::const_iterator last(updatedPhrase.end());
2392 for (; first != last; ++first)
2393 {
2394
2395 ucstring::size_type firstFun = text.find( ucstring(first->Identifier));
2396 if (firstFun == ucstring::npos)
2397 {
2398 nlwarning("Error can't patch %s: %s not found", filename.c_str(), first->Identifier.c_str());
2399 }
2400 else
2401 {
2402 ucstring::size_type lastFun = text.find( ucstring("}") , firstFun);
2403 if (lastFun == ucstring::npos)
2404 {
2405 nlwarning("Error can't patch %s: syntax error near %s", filename.c_str(), first->Identifier.c_str());
2406 }
2407 else
2408 {
2409 std::vector<TPhrase> param;
2410 param.push_back(*first);
2411
2412 ucstring before = text.substr(0,firstFun);
2413 ucstring str = preparePhraseFile2(param, false);
2414 ucstring after = text.substr(lastFun+1);
2415 text = "";
2416 text += before;
2417 text += str;
2418 text += after;
2419 }
2420 }
2421 }
2422 CI18N::writeTextFile( filename, text);
2423
2424 }
2425
2426 int updatePhraseWork()
2427 {
2428 std::string saveDir = diffDir + "update_"+ diffVersion + "/";
2429 vector<TPhrase> transPhrase;
2430 std::map<std::string, TPhrase> transPhraseMap;
2431 std::map<std::string, std::pair<uint64,uint64> > validClauseHashValue;
2432 std::map<std::string, std::pair<uint64, uint64> > validPhraseHashValue;
2433 std::vector< std::pair<ucstring, std::string> > outputResult;
2434
2435 if (!readPhraseFile(transDir+"phrase_wk.txt", transPhrase, false))
2436 {
2437 LOG("Error will loading file %s", (addDir+"phrase_"+Languages[0]+".txt").c_str());
2438 return 1;
2439 }
2440
2441 {
2442 std::vector<TPhrase>::const_iterator first(transPhrase.begin());
2443 std::vector<TPhrase>::const_iterator last(transPhrase.end());
2444 for (; first != last;++first)
2445 {
2446 transPhraseMap[first->Identifier] = *first;
2447 }
2448 }
2449
2450 preprocessTextFile(addDir+"phrase_wk.txt", outputResult);
2451
2452 uint firstFile = 0;
2453 uint lastFile = (uint)outputResult.size();
2454 for (; firstFile != lastFile ; ++firstFile)
2455 {
2456 ucstring doc = outputResult[firstFile].first;
2457 std::vector<TPhrase> phrases;
2458 readPhraseFileFromString(outputResult[firstFile].first, outputResult[firstFile].second, phrases, true);
2459
2460 std::vector<TPhrase>::iterator first(phrases.begin());
2461 std::vector<TPhrase>::iterator last(phrases.end());
2462 std::vector<TPhrase> updatedPhrases;
2463 for (; first != last; ++first)
2464 {
2465 if (transPhraseMap.find(first->Identifier) != transPhraseMap.end() )
2466 {
2467 TPhrase workPhrase = *first;
2468 TPhrase& transPhrase = transPhraseMap[first->Identifier];
2469 if (first->HashValue == transPhrase.HashValue)
2470 {
2471 uint64 oldHash = transPhrase.HashValue;
2472 uint64 newHash = STRING_MANAGER::makePhraseHash(transPhrase);
2473 if (newHash != transPhrase.HashValue)
2474 {
2475 //translation phrase_wk.txt has been manually changed
2476 validPhraseHashValue[transPhrase.Identifier] = std::pair<uint64, uint64>(newHash, oldHash);
2477 std::vector<TClause>::iterator firstClause ( transPhrase.Clauses.begin() );
2478 std::vector<TClause>::iterator lastClause ( transPhrase.Clauses.end() );
2479 for (; firstClause != lastClause; ++firstClause)
2480 {
2481 uint64 clauseHashValue = CI18N::makeHash(firstClause->Text);
2482
2483 validClauseHashValue[firstClause->Identifier] = std::pair<uint64, uint64>(clauseHashValue, firstClause->HashValue);
2484 firstClause->HashValue = clauseHashValue;
2485 }
2486 updatedPhrases.push_back(transPhrase);
2487 updatedPhrases.back().Comments.clear();
2488 }
2489 }
2490 }
2491 }
2492
2493
2494 std::string newFile = saveDir + outputResult[firstFile].second;
2495 std::string oldFile = outputResult[firstFile].second;
2496 CFile::createDirectoryTree(CFile::getPath(newFile));
2497 if ( CFile::copyFile(newFile, oldFile) )
2498 {
2499
2500 patchWorkFile(updatedPhrases, newFile);
2501 }
2502 else
2503 {
2504 nlwarning("Can't copy %s", newFile.c_str());
2505 }
2506
2507 }
2508
2509 updatePhraseHashValue(validPhraseHashValue, saveDir);
2510 updateClauseHashValue(validClauseHashValue, saveDir);
2511 return 0;
2512 }
2513
2514
2515
2516
2517 bool mergePhraseDiff2(vector<TPhrase> &phrases, const string &language, bool onlyTranslated, bool archiveDiff = false)
2518 {
2519 vector<string> diffs;
2520
2521 getPathContentFiltered(diffDir+"phrase_"+language+"_diff_", ".txt", diffs);
2522
2523 for (uint i=0; i<diffs.size(); ++i)
2524 {
2525 if (onlyTranslated)
2526 {
2527 // Check if the diff is translated
2528 ucstring text;
2529 CI18N::readTextFile(diffs[i], text, false, false, CI18N::LINE_FMT_LF);
2530 verifyVersion(text, 2);
2531 if (text.find(ucstring("DIFF NOT TRANSLATED")) != ucstring::npos)
2532 {
2533 LOG("Diff file [%s] is not translated, merging it later.\n", CFile::getFilename(diffs[i]).c_str());
2534 for (i=i+1; i<diffs.size(); ++i)
2535 LOG(" Merge of Diff file [%s] delayed.\n", CFile::getFilename(diffs[i]).c_str());
2536 return true;
2537 }
2538 }
2539
2540
2541
2542 // we found a diff file for the addition file.
2543 LOG("Adding %s diff as reference\n", diffs[i].c_str());
2544 vector<TPhrase> diff;
2545 if (!readPhraseFile2(diffs[i], diff, false))
2546 return false;
2547
2548
2549 mergePhraseDiff2Impl(phrases, diff);
2550
2551
2552 if (archiveDiff)
2553 {
2554 // move the diff file in the history dir
2555 CFile::moveFile((historyDir+CFile::getFilename(diffs[i])).c_str(), diffs[i].c_str());
2556 }
2557 }
2558
2559 return true;
2560 }
2561
2562
2563
2564 class CMakePhraseDiff2
2565 {
2566 public:
2567
2568 class CPhraseEqual
2569 {
2570
2571 public:
2572 CPhraseEqual(){}
2573
2574 bool operator()( const TPhrase& left, const TPhrase& right) const;
2575
2576 // bool clausesEqual( const std::vector<TClause>& left, const std::vector<TClause>& right) const;
2577
2578 // bool clauseEqual(const TClause& left, const TClause& right) const;
2579
2580 };
2581
2582 void run(const vector<TPhrase> &addition, vector<TPhrase> &reference, vector<TPhrase> &diff);
2583
2584 void onEquivalent(uint addIndex, uint refIndex, TPhraseDiffContext &context);
2585
2586 void onAdd(uint addIndex, uint refIndex, TPhraseDiffContext &context);
2587
2588 void onRemove(uint addIndex, uint refIndex, TPhraseDiffContext &context);
2589
2590 void onChanged(uint addIndex, uint refIndex, TPhraseDiffContext &context);
2591
2592
2593 };
2594
2595
2596
2597
2598
2599
2600 void CMakePhraseDiff2::run(const vector<TPhrase> &addition, vector<TPhrase> &reference, vector<TPhrase> &diff)
2601 {
2602
2603 TPhraseDiffContext context(addition, reference, diff);
2604
2605 std::set<std::string> phraseIdentifier;
2606 std::map<std::string, uint> mapAdd;
2607 std::map<std::string, uint> mapRef;
2608
2609 {
2610 uint first = 0;
2611 uint last = (uint)reference.size();
2612
2613 for ( ;first != last; ++first)
2614 {
2615 std::string Identifier(reference[first].Identifier);
2616 mapRef[Identifier] = first;
2617 phraseIdentifier.insert(Identifier);
2618 }
2619 }
2620
2621 {
2622 uint first = 0;
2623 uint last = (uint)addition.size();
2624
2625 for ( ;first != last; ++first)
2626 {
2627 std::string Identifier(addition[first].Identifier);
2628 mapAdd[Identifier] = first;
2629 phraseIdentifier.insert(Identifier);
2630 }
2631 }
2632
2633 if (mapAdd.size() != addition.size())
2634 {
2635 nlwarning("Phrases are defined more than once in works directory");
2636 }
2637
2638 if (mapAdd.size() != addition.size())
2639 {
2640 nlwarning("Phrases are defined more than once in translation directory");
2641 }
2642
2643
2644 std::set<std::string>::iterator first(phraseIdentifier.begin());
2645 std::set<std::string>::iterator last(phraseIdentifier.end());
2646
2647 for (; first != last; ++first)
2648 {
2649 if ( mapAdd.find(*first) != mapAdd.end()
2650 && mapRef.find(*first) != mapRef.end())
2651 {
2652
2653 if ( CPhraseEqual()(addition[mapAdd[*first]], reference[mapRef[*first]]) )
2654 {
2655 onEquivalent(mapAdd[*first], mapRef[*first], context);
2656 }
2657 else
2658 {
2659 onChanged(mapAdd[*first], mapRef[*first], context);
2660 }
2661 }
2662 else if ( mapAdd.find(*first) != mapAdd.end()
2663 && mapRef.find(*first) == mapRef.end())
2664 {
2665 onAdd(mapAdd[*first], 0, context);
2666 }
2667 else if ( mapAdd.find(*first) == mapAdd.end()
2668 && mapRef.find(*first) != mapRef.end())
2669 {
2670 onRemove(0, mapRef[*first], context);
2671 }
2672
2673 }
2674
2675 }
2676
2677
2678 void CMakePhraseDiff2::onEquivalent(uint addIndex, uint refIndex, TPhraseDiffContext &context)
2679 {
2680 // nothing to do
2681 }
2682 void CMakePhraseDiff2::onAdd(uint addIndex, uint refIndex, TPhraseDiffContext &context)
2683 {
2684 TPhrase phrase = context.Addition[addIndex];
2685 char temp[1024];
2686 sprintf(temp, "// DIFF ADD");
2687 phrase.Comments = ucstring(temp) + nl + phrase.Comments;
2688
2689 nlinfo("Added %s at %u", phrase.Identifier.c_str(), addIndex);
2690 context.Diff.push_back(phrase);
2691 }
2692
2693 void CMakePhraseDiff2::onRemove(uint addIndex, uint refIndex, TPhraseDiffContext &context)
2694 {
2695 TPhrase phrase = context.Reference[refIndex];
2696 char temp[1024];
2697 sprintf(temp, "// DIFF REMOVED");
2698 // NB : on vire les commentaires car il pourrai contenir des merdes..
2699 phrase.Comments = ucstring(temp) + nl;
2700 for (uint i=0; i<phrase.Clauses.size(); ++i)
2701 phrase.Clauses[i].Comments.erase();
2702
2703 nlinfo("Removed %s at %u", phrase.Identifier.c_str(), addIndex);
2704 context.Diff.push_back(phrase);
2705 }
2706
2707 void CMakePhraseDiff2::onChanged(uint addIndex, uint refIndex, TPhraseDiffContext &context)
2708 {
2709 ucstring chg;
2710 // check what is changed.
2711 if (context.Addition[addIndex].Parameters != context.Reference[refIndex].Parameters)
2712 chg += "// Parameter list changed." + nl;
2713 if (context.Addition[addIndex].Clauses.size() != context.Reference[refIndex].Clauses.size())
2714 chg += "// Clause list changed." + nl;
2715 else
2716 {
2717 for (uint i=0; i<context.Addition[addIndex].Clauses.size(); ++i)
2718 {
2719 if (context.Addition[addIndex].Clauses[i].Identifier != context.Reference[refIndex].Clauses[i].Identifier)
2720 chg += ucstring("// Clause ") + toString(i) + " : identifier changed." + nl;
2721 else if (context.Addition[addIndex].Clauses[i].Conditions != context.Reference[refIndex].Clauses[i].Conditions)
2722 chg += ucstring("// Clause ") + toString(i) + " : condition changed." + nl;
2723 else if (context.Addition[addIndex].Clauses[i].Text != context.Reference[refIndex].Clauses[i].Text)
2724 chg += ucstring("// Clause ") + toString(i) + " : text changed." + nl;
2725 }
2726 }
2727
2728 if (chg.empty())
2729 {
2730 chg = ucstring("// WARNING : Hash code changed ! check translation workflow.") + nl;
2731 }
2732 nldebug("Changed detected : %s", chg.toString().c_str());
2733
2734 // changed element
2735 TPhrase phrase = context.Addition[addIndex];
2736 vector<TPhrase> tempV;
2737 tempV.push_back(context.Reference[refIndex]);
2738 ucstring tempT = preparePhraseFile(tempV, false);
2739 CI18N::removeCComment(tempT);
2740 phrase.Comments = ucstring("// DIFF CHANGED ") + toString(addIndex) + nl + phrase.Comments;
2741 phrase.Comments = phrase.Comments + ucstring("/* OLD VALUE : [" + nl) + tabLines(1, tempT) + nl + "] */" + nl;
2742 phrase.Comments = phrase.Comments + chg;
2743
2744 nlinfo("Changed %s at %u", phrase.Identifier.c_str(), addIndex);
2745 context.Diff.push_back(phrase);
2746 }
2747
2748
2749 bool CMakePhraseDiff2::CPhraseEqual::operator()( const TPhrase& left, const TPhrase& right) const
2750 {
2751 bool identifierOk = left.Identifier == right.Identifier;
2752 // bool parameterOk = left.Parameters == right.Parameters;
2753 // bool commentsOk = left.Comments == right.Comments;
2754 // bool clausesOk = clausesEqual(left.Clauses, right.Clauses);
2755 bool hashOk = left.HashValue== right.HashValue;
2756
2757 return identifierOk && hashOk;// && parameterOk && clausesOk;
2758
2759 }
2760 /*
2761 bool CMakePhraseDiff2::CPhraseEqual::clausesEqual( const std::vector<TClause>& left, const std::vector<TClause>& right) const
2762 {
2763 std::vector<TClause>::const_iterator first1(left.begin());
2764 std::vector<TClause>::const_iterator last1(left.end());
2765 std::vector<TClause>::const_iterator first2(right.begin());
2766
2767 if (left.size() != right.size()) return false;
2768
2769 for ( ; first1 != last1 && !clauseEqual(*first1, *first2); ++first1, ++first2){}
2770
2771 return first1 == last1;
2772 }
2773
2774 bool CMakePhraseDiff2::CPhraseEqual::clauseEqual(const TClause& left, const TClause& right) const
2775 {
2776 return left.Identifier != right.Identifier
2777 && left.Conditions != right.Conditions
2778 && left.Text != right.Text
2779 && left.Comments != right.Comments
2780 && left.HashValue != right.HashValue;
2781 }
2782
2783 */
2784
2785
2786
2787 int makePhraseDiff2(int argc, char *argv[])
2788 {
2789 // Generate the diff file from phrase_<lang>.txt compared to the same file in translated.
2790 // The diff is generated only from the reference language for and all the languages
2791
2792 LOG("Generating phrase diffs\nLoading the working file for language %s\n", Languages[0].c_str());
2793
2794
2795 vector<TPhrase> addition;
2796
2797 // read addition
2798 if (!readPhraseFile(addDir+"phrase_"+Languages[0]+".txt", addition, true))
2799 {
2800 LOG("Error will loading file %s", (addDir+"phrase_"+Languages[0]+".txt").c_str());
2801 return 1;
2802 }
2803
2804 for (uint l =0; l<Languages.size(); ++l)
2805 {
2806 LOG("Diffing with language %s...\n", Languages[l].c_str());
2807 if (l == 1)
2808 {
2809 addition.clear();
2810 // read the language 0 translated version as addition for other language
2811 if (!readPhraseFile(transDir+"phrase_"+Languages[0]+".txt", addition, true))
2812 {
2813 LOG("Error will loading file %s", (addDir+"phrase_"+Languages[0]+".txt").c_str());
2814 return 1;
2815 }
2816 }
2817 vector<TPhrase> reference;
2818 // read the reference file
2819 if (!readPhraseFile(transDir+"phrase_"+Languages[l]+".txt", reference, false))
2820 {
2821 LOG("Error will loading file %s", (transDir+"phrase_"+Languages[l]+".txt").c_str());
2822 return 1;
2823 }
2824
2825 if (!mergePhraseDiff2(reference, Languages[l], false))
2826 {
2827 LOG("Error will merging phrase diff for language %s\n", Languages[l].c_str());
2828 return 1;
2829 }
2830
2831 // compare the reference an addition file, remove any equivalent strings.
2832 uint addCount=0, refCount=0;
2833 vector<TPhrase> diff;
2834
2835 CMakePhraseDiff2 differ;
2836 differ.run(addition, reference, diff);
2837
2838 if (diff.empty())
2839 {
2840 LOG("No difference for language %s\n", Languages[l].c_str());
2841 }
2842 else
2843 {
2844 LOG("Writing difference file for language %s\n", Languages[l].c_str());
2845 ucstring text;
2846 text += "// DIFF_VERSION 2\n";
2847 text += preparePhraseFile(diff, false);
2848 // add the tag for non translation
2849 text += nl + ucstring ("// REMOVE THE FOLOWING LINE WHEN TRANSLATION IS DONE") + nl + ucstring("// DIFF NOT TRANSLATED") + nl;
2850 CI18N::writeTextFile(diffDir+"phrase_"+Languages[l]+"_diff_"+diffVersion+".txt", text);
2851 }
2852 }
2853
2854 return 0;
2855 }
2856
2857
2858
2859 int forgetPhraseDiff(int argc, char *argv[])
2860 {
2861 // merge all the phrase diff back into there repective translated phrase.
2862
2863 LOG("forgeting phrase diffs\n");
2864
2865 std::string basename("phrase_"+Languages[0]);
2866 string filename = transDir+basename+".txt";
2867 // build the addition diff
2868 vector<TPhrase> reference;
2869
2870
2871 if (!readPhraseFile(transDir+basename+".txt", reference, false))
2872 {
2873 LOG("Error will loading file %s", (transDir+basename+".txt").c_str());
2874 return 1;
2875 }
2876 //assert only change
2877
2878 std::vector<std::string> diffs;
2879 getPathContentFiltered(diffDir+"phrase_wk_diff_", ".txt", diffs);
2880 std::vector<TPhrase> newPhrase;
2881 for (uint i=0; i<diffs.size(); ++i)
2882 {
2883 // we found a diff file for the addition file.
2884 LOG("Adding %s diff as reference\n", diffs[i].c_str());
2885 vector<TPhrase> subDiff;
2886 if (!readPhraseFile2(diffs[i], subDiff, false))
2887 return false;
2888 std::copy (subDiff.begin (), subDiff.end (), std::back_inserter (newPhrase));
2889 }
2890
2891 // a optimiser par une map
2892 std::map<std::string, std::pair<uint64, uint64> > validClauseHashValue;
2893 std::map<std::string, std::pair<uint64, uint64> > validPhraseHashValue;
2894 for (uint i=0; i < newPhrase.size() ; ++i)
2895 {
2896 for (uint j=0; j < reference.size() ; ++j)
2897 {
2898 if (newPhrase[i].Identifier == reference[j].Identifier)
2899 {
2900
2901
2902 uint64 newPhraseHash = STRING_MANAGER::makePhraseHash( newPhrase[i] );
2903 uint64 oldPhraseHash = reference[j].HashValue;
2904 validPhraseHashValue[newPhrase[i].Identifier] = std::pair<uint64, uint64>(newPhraseHash, oldPhraseHash);
2905
2906 for (uint k=0; k < newPhrase[i].Clauses.size() ; ++k)
2907 {
2908 if (reference[j] .Clauses.size() != newPhrase[i].Clauses.size())
2909 {
2910 nlwarning("Want to forget minor update but phrase %s changes too much. The number of clauses has changed.", newPhrase[i].Identifier.c_str() );
2911 exit(-1);
2912 }
2913 const TClause& newClause = newPhrase[i].Clauses[k];
2914 const TClause& oldClause = reference[j].Clauses[k];
2915
2916 if (!newClause.Identifier.empty() )
2917 {
2918 if (newClause.Identifier != oldClause.Identifier)
2919 {
2920 nlwarning("Want to forget minor update but phrase %s changes too much. Clauses order or clause identifier changed (%s).", newPhrase[i].Identifier.c_str(), newClause.Identifier.c_str());
2921 exit(-1);
2922 }
2923 uint64 newClauseHashValue = CI18N::makeHash(newClause.Text);
2924 uint64 oldClauseHashValue = CI18N::makeHash(oldClause.Text);
2925 validClauseHashValue[ newClause.Identifier ] = std::pair<uint64, uint64>(newClauseHashValue, oldClauseHashValue);
2926 }
2927 }
2928 }
2929 }
2930 }
2931
2932
2933 if (!mergePhraseDiff2(reference, Languages[0], true, false))
2934 {
2935 LOG("Error will merging phrase diff");
2936 return 1;
2937 }
2938 ucstring str = preparePhraseFile(reference, true);
2939 CI18N::writeTextFile(transDir+basename+".txt", str);
2940
2941
2942 updatePhraseHashValue(validPhraseHashValue);
2943 // updateClauseHashValue(validClauseHashValue);
2944
2945
2946 for (uint i=0; i<diffs.size(); ++i)
2947 {
2948 std::string diffHistory = historyDir + CFile::getFilename(diffs[i]);
2949 CFile::moveFile(diffHistory.c_str(), diffs[i].c_str());
2950 }
2951 return 0;
2952 }
2953
2954
2955 void preprocessTextFile(const std::string &filename,
2956 std::vector< std::pair<ucstring, std::string> > & outputResult
2957 )
2958
2959 {
2960 //nlinfo("preprocessing %s", filename.c_str());
2961 ucstring result;
2962 std::string fullName;
2963 fullName = filename;
2964
2965 if (fullName.empty())
2966 return;
2967
2968 NLMISC::CIFile file;
2969
2970 /// Open a file for reading. false if failed. close() if a file was opened.
2971 if (!file.open (fullName))
2972 {
2973 nlwarning("Can't open %s", fullName.c_str());
2974 return ;
2975 }
2976
2977
2978
2979 // Fast read all the text in binary mode.
2980 std::string text;
2981 text.resize(file.getFileSize());
2982 file.serialBuffer((uint8*)(&text[0]), (uint)text.size());
2983
2984
2985 // Transform the string in ucstring according to format header
2986 if (!text.empty())
2987 CI18N::readTextBuffer((uint8*)&text[0], (uint)text.size(), result);
2988
2989
2990
2991 ucstring final;
2992 // parse the file, looking for preprocessor command.
2993 ucstring::size_type pos = 0;
2994 ucstring::size_type lastPos = 0;
2995 ucstring includeCmd("#include");
2996 ucstring current;
2997
2998 while ( pos != ucstring::npos)
2999 {
3000 pos = result.find(ucstring("\n"), pos);
3001 if (pos != ucstring::npos) { ++pos; }
3002
3003 ucstring line( result.substr(lastPos, pos - lastPos) );
3004
3005
3006 if ( line.find(includeCmd) != ucstring::npos)
3007 {
3008 ucstring::size_type firstFilename = line.find(ucstring("\""));
3009 ucstring::size_type lastFilename = line.find(ucstring("\""), firstFilename+1);
3010
3011 ucstring name = line.substr(firstFilename +1, lastFilename - firstFilename -1);
3012 string subFilename = name.toString();
3013
3014 if (!CFile::fileExists(subFilename))
3015 {
3016 // try to open the include file relative to current file
3017 subFilename = CFile::getPath(filename)+subFilename;
3018
3019 if (!CFile::fileExists(subFilename))
3020 {
3021 nlwarning("Unable to open %s", subFilename.c_str());
3022 subFilename.clear();
3023 }
3024 }
3025
3026 preprocessTextFile(subFilename, outputResult);
3027 }
3028 else
3029 {
3030 current += line;
3031 }
3032 lastPos = pos;
3033
3034 }
3035
3036 outputResult.push_back( std::pair<ucstring, std::string> ( current, fullName ) );
3037 }
3038
3039 int mergePhraseDiff(int argc, char *argv[])
3040 {
3041 // merge all the phrase diff back into there repective translated phrase.
3042 uint l;
3043
3044 LOG("Merging phrase diffs\n");
3045
3046 for (l=0; l<Languages.size(); ++l)
3047 {
3048 LOG("Merging for language %s...\n", Languages[l].c_str());
3049 std::string basename("phrase_"+Languages[l]);
3050 string filename = transDir+basename+".txt";
3051 // build the addition diff
3052 vector<TPhrase> reference;
3053
3054 if (!readPhraseFile(transDir+basename+".txt", reference, false))
3055 {
3056 LOG("Error will loading file %s", (transDir+basename+".txt").c_str());
3057 return 1;
3058 }
3059
3060 if (!mergePhraseDiff(reference, Languages[l], true, true))
3061 {
3062 LOG("Error will merging phrase diff");
3063 return 1;
3064
3065 }
3066
3067 ucstring str = preparePhraseFile(reference, true);
3068
3069 {
3070 // backup the original file
3071 ucstring old;
3072 CI18N::readTextFile(filename, old, true, false, CI18N::LINE_FMT_LF);
3073 if (old != str)
3074 CFile::moveFile((historyDir+CFile::getFilenameWithoutExtension(filename)+"_"+diffVersion+"."+CFile::getExtension(filename)).c_str(), filename.c_str());
3075 }
3076
3077 CI18N::writeTextFile(transDir+basename+".txt", str);
3078
3079 }
3080
3081 return 0;
3082 }
3083
3084
3085 int injectClause()
3086 {
3087 uint l;
3088
3089 LOG("Update translation from clauses.\n");
3090
3091 for (l=0; l<Languages.size(); ++l)
3092 {
3093
3094 nlinfo("Update phrase %s", Languages[l].c_str());
3095 vector<TStringInfo> clauses;
3096 vector<TPhrase> phrases;
3097
3098 // load the clause file
3099 std::string clausePath( transDir+"clause_"+Languages[l]+".txt" );
3100 if (!loadStringFile(clausePath, clauses, false))
3101 {
3102 LOG("Error will loading file %s", clausePath.c_str());
3103 return 1;
3104 }
3105
3106 // load the phrase file
3107 std::string phrasePath( transDir+"phrase_"+Languages[l]+".txt" );
3108 if (!readPhraseFile(phrasePath, phrases, false))
3109 {
3110 LOG("Error will loading file %s", phrasePath.c_str());
3111 return 1;
3112 }
3113
3114 vector<TPhrase>::iterator first(phrases.begin());
3115 vector<TPhrase>::iterator last(phrases.end());
3116
3117 for ( ; first != last; ++first)
3118 {
3119 vector<TClause>::iterator firstClause( first->Clauses.begin());
3120 vector<TClause>::iterator lastClause( first->Clauses.end());
3121 for ( ; firstClause != lastClause; ++firstClause)
3122 {
3123 vector<TStringInfo>::iterator firstRefClause(clauses.begin());
3124 vector<TStringInfo>::iterator lastRefClause(clauses.end());
3125 for ( ; firstRefClause != lastRefClause ; ++firstRefClause)
3126 {
3127 if (firstClause->Identifier == firstRefClause->Identifier && firstClause->Text != firstRefClause->Text)
3128 {
3129 firstClause->Text = firstRefClause->Text;
3130 firstClause->HashValue = CI18N::makeHash(firstClause->Text);
3131 firstRefClause->HashValue = firstClause->HashValue;
3132
3133 nlinfo("update clause %s from clause file %s.", firstClause->Identifier.c_str(), clausePath.c_str());
3134 }
3135 }
3136 }
3137 }
3138
3139 std::string desDir(diffDir + "inject_clause_" + diffVersion + "/");
3140 CFile::createDirectoryTree(desDir+ CFile::getPath(phrasePath));
3141 ucstring str = preparePhraseFile(phrases, true);
3142 CI18N::writeTextFile(desDir + phrasePath, str);
3143
3144 str = prepareStringFile(clauses, true);
3145 CI18N::writeTextFile(desDir + clausePath, str);
3146 }
3147
3148 return 0;
3149 }
3150
3151
3152
3153 int main(int argc, char *argv[])
3154 {
3155 NLMISC::CApplicationContext context;
3156 /* createDebug();
3157 CStdDisplayer *display = new CStdDisplayer;
3158 NLMISC::InfoLog->addDisplayer(display);
3159 NLMISC::WarningLog->addDisplayer(display);
3160 NLMISC::ErrorLog->addDisplayer(display);
3161 */
3162
3163 /* for (uint i=0; i<20; ++i)
3164 {
3165 uint64 hash = makeHash(ucstring("Bonjour le monde !"));
3166 nldebug("%s", hashToString(hash).c_str());
3167 hash = makeHash(ucstring("Une autre clef"));
3168 nldebug("%s", hashToString(hash).c_str());
3169 }
3170 */
3171
3172 if (argc < 2)
3173 {
3174 showUsage(argv[0]);
3175 return 1;
3176 }
3177 std::string argv1(argv[1]);
3178
3179 // create the diff version.
3180 char temp[16];
3181 sprintf(temp, "%8.8X", (uint) ::time(NULL));
3182 diffVersion = temp;
3183
3184 if (strcmp(argv[1], "make_work") == 0)
3185 {
3186 return makeWork();
3187 }
3188
3189 // generic worksheet comparison
3190 if (strcmp(argv[1], "make_worksheet_diff") == 0)
3191 {
3192 if (argc != 3)
3193 {
3194 showUsage(argv[0]);
3195 return 1;
3196 }
3197
3198 return makeWorksheetDiff(argc, argv, argv[2], argv[2], true);
3199 }
3200 else if (strcmp(argv[1], "merge_worksheet_diff") == 0)
3201 {
3202 if (argc != 3)
3203 {
3204 showUsage(argv[0]);
3205 return 1;
3206 }
3207
3208 return mergeWorksheetDiff(argc, argv, argv[2], argv[2]);
3209 }
3210 else if (strcmp(argv[1], "crop_lines") == 0)
3211 {
3212 if (argc != 4)
3213 {
3214 showUsage(argv[0]);
3215 return 1;
3216 }
3217
3218 uint nbLines;
3219 NLMISC::fromString(argv[3], nbLines);
3220
3221 cropLines(argv[2], nbLines);
3222
3223 return 0;
3224 }
3225 else if (strcmp(argv[1], "extract_bot_names") == 0)
3226 return extractBotNames(argc, argv);
3227 else if (strcmp(argv[1], "extract_new_sheet_names") == 0)
3228 return extractNewSheetNames(argc, argv);
3229
3230
3231
3232
3233 if (argc != 2)
3234 {
3235 showUsage(argv[0]);
3236 return 1;
3237 }
3238
3239 // if (strcmp(argv[1], "yann") == 0)
3240 // return mergeYannTaf();
3241
3242
3243 string currentPath("./");
3244 CPath::addSearchPath(currentPath+addDir, true, false);
3245 CPath::addSearchPath(currentPath+diffDir, true, false);
3246 // CPath::addSearchPath(currentPath+transDir, true, false);
3247 if (readLanguages() != 0)
3248 {
3249 LOG("Error will loading language file (language.txt)");
3250 return 1;
3251 }
3252
3253 if (strcmp(argv[1], "make_string_diff") == 0)
3254 return makeStringDiff(argc, argv, "");
3255 else if (strcmp(argv[1], "merge_string_diff") == 0)
3256 return mergeStringDiff(argc, argv, "");
3257 else if (strcmp(argv[1], "clean_string_diff") == 0)
3258 return cleanStringDiff(argc, argv, "");
3259
3260 else if (strcmp(argv[1], "make_r2_string_diff") == 0)
3261 return makeStringDiff(argc, argv, "r2_");
3262 else if (strcmp(argv[1], "merge_r2_string_diff") == 0)
3263 return mergeStringDiff(argc, argv, "r2_");
3264 else if (strcmp(argv[1], "clean_r2_string_diff") == 0)
3265 return cleanStringDiff(argc, argv, "r2_");
3266
3267 else if (argv1 == "make_phrase_diff_old")
3268 return makePhraseDiff(argc, argv);
3269 else if (argv1 == "merge_phrase_diff_old")
3270 return mergePhraseDiff(argc, argv, 1);
3271
3272
3273 else if (argv1 == "make_phrase_diff")
3274 return makePhraseDiff2(argc, argv);
3275 else if (argv1 == "merge_phrase_diff")
3276 return mergePhraseDiff(argc, argv, 2);
3277 else if (argv1 == "forget_phrase_diff")
3278 return forgetPhraseDiff(argc, argv);
3279 else if (argv1 == "update_phrase_work")
3280 return updatePhraseWork();
3281 else if (argv1 == "clean_phrase_diff")
3282 return cleanPhraseDiff(argc, argv);
3283
3284 else if (argv1 == "inject_clause")
3285 return injectClause();
3286
3287 else if (argv1 == "sort_trans_phrase")
3288 return sortTransPhrase();
3289
3290 else if (strcmp(argv[1], "make_clause_diff") == 0)
3291 return makeClauseDiff(argc, argv);
3292 else if (strcmp(argv[1], "merge_clause_diff") == 0)
3293 return mergeClauseDiff(argc, argv);
3294 else if (argv1 == "clean_clause_diff")
3295 return cleanClauseDiff(argc, argv);
3296
3297 else if (strcmp(argv[1], "make_words_diff") == 0)
3298 return makeWordsDiff(argc, argv);
3299 else if (strcmp(argv[1], "merge_words_diff") == 0)
3300 return mergeWordsDiff(argc, argv);
3301 else if (strcmp(argv[1], "clean_words_diff") == 0)
3302 return cleanWordsDiff(argc, argv);
3303
3304 else if (strcmp(argv[1], "recup_around") == 0)
3305 return recupAround(argc, argv);
3306 else if (strcmp(argv[1], "add_string_number") == 0)
3307 return addStringNumber();
3308
3309
3310 return -1;
3311
3312 }
3313
3314
3315
3316 int addStringNumber()
3317 {
3318 vector<TStringInfo> strings;
3319
3320 LOG("Generating string diffs\nLoading the working file for language %s\n", Languages[0].c_str());
3321 // load the addition file
3322 std::string addFile(Languages[0]+".uxt");
3323 if (!loadStringFile(addDir+addFile, strings, true))
3324 {
3325 LOG("Error loading file %s\n", (addDir+addFile).c_str());
3326 return 1;
3327 }
3328
3329 ucstring str = prepareStringFile(strings, false);
3330
3331 string filename = addDir+Languages[0]+".uxt";
3332 CI18N::writeTextFile(filename, str);
3333
3334 return 0;
3335 }
3336
3337