nel/src/misc/diff_tool.cpp

   1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This source file has been modified by the following contributors:
   5 // Copyright (C) 2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
   6 //
   7 // This program is free software: you can redistribute it and/or modify
   8 // it under the terms of the GNU Affero General Public License as
   9 // published by the Free Software Foundation, either version 3 of the
  10 // License, or (at your option) any later version.
  11 //
  12 // This program is distributed in the hope that it will be useful,
  13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 // GNU Affero General Public License for more details.
  16 //
  17 // You should have received a copy of the GNU Affero General Public License
  18 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 #include "stdmisc.h"
  21
  22 #include "nel/misc/diff_tool.h"
  23 #include "nel/misc/path.h"
  24
  25 using namespace NLMISC;
  26 using namespace std;
  27
  28 #ifdef DEBUG_NEW
  29         #define new DEBUG_NEW
  30 #endif
  31
  32 namespace STRING_MANAGER
  33 {
  34
  35 uint64 makePhraseHash(const TPhrase &phrase)
  36 {
  37         ucstring text;
  38         text = phrase.Parameters;
  39         for (uint i=0; i<phrase.Clauses.size(); ++i)
  40         {
  41                 text += phrase.Clauses[i].Conditions;
  42                 text += phrase.Clauses[i].Identifier;
  43                 text += phrase.Clauses[i].Text;
  44         }
  45
  46         return CI18N::makeHash(text);
  47 }
  48
  49
  50
  51
  52 bool parseHashFromComment(const ucstring &comments, uint64 &hashValue)
  53 {
  54         string str = comments.toString();
  55
  56         string::size_type pos = str.find("HASH_VALUE ");
  57         if (pos == string::npos)
  58                 return false;
  59
  60         string hashStr = str.substr(pos + 11, 16);
  61
  62         hashValue = CI18N::stringToHash(hashStr);
  63         return true;
  64 }
  65
  66
  67 uint32 countLine(const ucstring &text, const ucstring::const_iterator upTo)
  68 {
  69         uint32 ret = 1;
  70         ucstring::const_iterator first(text.begin());
  71
  72         for (; first != upTo; ++first)
  73         {
  74                 if (*first == '\n')
  75                         ret++;
  76         }
  77
  78         return ret;
  79 }
  80
  81 bool loadStringFile(const std::string filename, vector<TStringInfo> &stringInfos, bool forceRehash, ucchar openMark, ucchar closeMark, bool specialCase)
  82 {
  83 /*      uint8 *buffer = 0;
  84         uint    size;
  85
  86         try
  87         {
  88                 CIFile fp(filename);
  89                 size = fp.getFileSize();
  90                 buffer = new uint8[size];
  91                 fp.serialBuffer(buffer, size);
  92         }
  93         catch(const Exception &e)
  94         {
  95                 nlinfo("Can't open file [%s] (%s)\n", filename.c_str(), e.what());
  96                 return true;
  97         }
  98 */
  99 /*      FILE *fp = nlfopen(filename, "rb");
 100
 101         if (fp == NULL)
 102         {
 103                 nlinfo("Can't open file [%s]\n", filename.c_str());
 104                 if (buffer != 0)
 105                         delete [] buffer;
 106                 return true;
 107         }
 108
 109         // move to end of file
 110         fseek(fp, 0, SEEK_END);
 111
 112         fpos_t  pos;
 113         fgetpos(fp, &pos);
 114
 115         uint8 *buffer = new uint8[uint(pos)];
 116
 117         rewind(fp);
 118         uint size = fread(buffer, 1, uint(pos), fp);
 119         fclose (fp);
 120 */
 121         ucstring text;
 122
 123         CI18N::readTextFile(filename, text, false, true, CI18N::LINE_FMT_LF);
 124 //      CI18N::readTextBuffer(buffer, size, text);
 125 //      delete [] buffer;
 126
 127         // ok, parse the file now.
 128         ucstring::const_iterator first(text.begin()), last(text.end());
 129         std::string lastLabel("nothing");
 130
 131         while (first != last)
 132         {
 133                 TStringInfo si;
 134                 CI18N::skipWhiteSpace(first, last, &si.Comments);
 135
 136                 if (first == last)
 137                 {
 138                         // check if there is only swap command remaining in comment
 139                         if (si.Comments.find(ucstring("// DIFF SWAP ")) != ucstring::npos)
 140                         {
 141                                 stringInfos.push_back(si);
 142                         }
 143                         break;
 144                 }
 145
 146                 // try to read a #fileline preprocessor command
 147                 if (CI18N::matchToken("#fileline", first, last))
 148                 {
 149                         // for now, just skip
 150                         uint32 lineCounter =0;  // we count line another way
 151                         CI18N::skipLine(first, last, lineCounter);
 152
 153                         // begin parse of next line
 154                         continue;
 155                 }
 156
 157                 if (!CI18N::parseLabel(first, last, si.Identifier))
 158                 {
 159                         uint32 line = countLine(text, first);
 160                         nlwarning("DT: Fatal : In '%s', line %u: Invalid label after '%s'",
 161                                 filename.c_str(),
 162                                 line,
 163                                 lastLabel.c_str());
 164                         return false;
 165                 }
 166                 lastLabel = si.Identifier;
 167
 168                 CI18N::skipWhiteSpace(first, last, &si.Comments);
 169
 170                 if (!CI18N::parseMarkedString(openMark, closeMark, first, last, si.Text))
 171                 {
 172                         uint32 line = countLine(text, first);
 173                         nlwarning("DT: Fatal : In '%s', line %u: Invalid text value for label %s",
 174                                 filename.c_str(),
 175                                 line,
 176                                 lastLabel.c_str());
 177                         return false;
 178                 }
 179
 180                 if (specialCase)
 181                 {
 182                         CI18N::skipWhiteSpace(first, last, &si.Comments);
 183
 184                         if (!CI18N::parseMarkedString(openMark, closeMark, first, last, si.Text2))
 185                         {
 186                                 uint32 line = countLine(text, first);
 187                                 nlwarning("DT: Fatal: In '%s' line %u: Invalid text2 value label %s",
 188                                         filename.c_str(),
 189                                         line,
 190                                         lastLabel.c_str());
 191                                 return false;
 192                         }
 193
 194                 }
 195
 196                 if (forceRehash || !parseHashFromComment(si.Comments, si.HashValue))
 197                 {
 198                         // compute the hash value from text.
 199                         si.HashValue = CI18N::makeHash(si.Text);
 200 //                      nldebug("Generating hash for %s as %s", si.Identifier.c_str(), CI18N::hashToString(si.HashValue).c_str());
 201                 }
 202                 else
 203                 {
 204 //                      nldebug("Comment = [%s]", si.Comments.toString().c_str());
 205 //                      nldebug("Retrieving hash for %s as %s", si.Identifier.c_str(), CI18N::hashToString(si.HashValue).c_str());
 206                 }
 207                 stringInfos.push_back(si);
 208         }
 209
 210
 211         // check identifier uniqueness
 212         {
 213                 bool error = false;
 214                 set<string>     unik;
 215                 set<string>::iterator it;
 216                 for (uint i=0; i<stringInfos.size(); ++i)
 217                 {
 218                         it = unik.find(stringInfos[i].Identifier);
 219                         if (it != unik.end())
 220                         {
 221                                 nlwarning("DT: loadStringFile : identifier '%s' exist twice", stringInfos[i].Identifier.c_str() );
 222                                 error = true;
 223                         }
 224                         else
 225                                 unik.insert(stringInfos[i].Identifier);
 226
 227                 }
 228                 if (error)
 229                         return false;
 230         }
 231
 232         return true;
 233 }
 234
 235
 236 ucstring prepareStringFile(const vector<TStringInfo> &strings, bool removeDiffComments, bool noDiffInfo)
 237 {
 238         string diff;
 239
 240         vector<TStringInfo>::const_iterator first(strings.begin()), last(strings.end());
 241         for (; first != last; ++first)
 242         {
 243                 string str;
 244                 const TStringInfo &si = *first;
 245                 string comment = si.Comments.toUtf8();
 246                 vector<string> lines;
 247                 explode(comment, string("\n"), lines, true);
 248
 249                 uint i;
 250                 for (i=0; i<lines.size(); ++i)
 251                 {
 252                         if (removeDiffComments)
 253                         {
 254                                 if (lines[i].find("// DIFF ") != string::npos)
 255                                 {
 256                                         lines.erase(lines.begin()+i);
 257                                         --i;
 258                                         continue;
 259                                 }
 260                         }
 261                         if (lines[i].find("// INDEX ") != string::npos)
 262                         {
 263                                 lines.erase(lines.begin()+i);
 264                                 --i;
 265                         }
 266                         else if (lines[i].find("// HASH_VALUE ") != string::npos)
 267                         {
 268                                 lines.erase(lines.begin()+i);
 269                                 --i;
 270                         }
 271                 }
 272
 273                 comment.erase();
 274                 for (i=0; i<lines.size(); ++i)
 275                 {
 276                         comment += lines[i] + "\n";
 277                 }
 278                 si.Comments = ucstring(comment);
 279
 280                 str = comment;
 281                 if (!si.Identifier.empty() || !si.Text.empty())
 282                 {
 283                         // add hash value comment if needed
 284 //                      if (si.Comments.find(ucstring("// HASH_VALUE ")) == ucstring::npos)
 285                         if (!noDiffInfo)
 286                         {
 287                                 str += "// HASH_VALUE " + CI18N::hashToString(si.HashValue) + "\n";
 288                                 str += "// INDEX " + NLMISC::toString("%u", first-strings.begin()) + "\n";
 289                         }
 290                         str += si.Identifier + '\t';
 291
 292                         string text = CI18N::makeMarkedString('[', ']', si.Text).toUtf8();
 293                         string text2;
 294                         // add new line and tab after each \n tag
 295                         string::size_type pos;
 296                         while ((pos = text.find("\\n")) != string::npos)
 297                         {
 298                                 text2 += text.substr(0, pos+2) + "\n\t";
 299                                 text = text.substr(pos+2);
 300                         }
 301                         text2 += text;//.substr(0, pos+2);
 302                         str += text2 + "\n\n";
 303 //                      str += CI18N::makeMarkedString('[', ']', si.Text) + nl + nl;
 304                 }
 305
 306 //              nldebug("Adding string [%s]", str.toString().c_str());
 307                 diff += str;
 308         }
 309
 310         return ucstring::makeFromUtf8(diff);
 311 }
 312
 313
 314 bool readPhraseFile(const std::string &filename, vector<TPhrase> &phrases, bool forceRehash)
 315 {
 316         ucstring doc;
 317
 318         CI18N::readTextFile(filename, doc, false, true, CI18N::LINE_FMT_LF);
 319
 320         return readPhraseFileFromString(doc, filename, phrases, forceRehash);
 321 }
 322
 323 bool readPhraseFileFromString(ucstring const& doc, const std::string &filename, vector<TPhrase> &phrases, bool forceRehash)
 324 {
 325         std::string lastRead("nothing");
 326
 327         ucstring::const_iterator first(doc.begin()), last(doc.end());
 328         while (first != last)
 329         {
 330                 TPhrase phrase;
 331                 // parse the phrase
 332                 CI18N::skipWhiteSpace(first, last, &phrase.Comments);
 333
 334                 if (first == last)
 335                 {
 336                         if (!phrase.Comments.empty())
 337                         {
 338                                 // push the resulting comment
 339                                 phrases.push_back(phrase);
 340                         }
 341                         break;
 342                 }
 343
 344                 // try to read a #fileline preprocessor command
 345                 if (CI18N::matchToken("#fileline", first, last))
 346                 {
 347                         // for now, just skip
 348                         uint32 lineCounter =0;  // we count line another way
 349                         CI18N::skipLine(first, last, lineCounter);
 350
 351                         // begin parse of next line
 352                         continue;
 353                 }
 354
 355                 if (!CI18N::parseLabel(first, last, phrase.Identifier))
 356                 {
 357                         uint32 line = countLine(doc, first);
 358                         nlwarning("DT: In '%s' line %u: Error parsing phrase identifier after %s\n",
 359                                 filename.c_str(),
 360                                 line,
 361                                 lastRead.c_str());
 362                         return false;
 363                 }
 364 //              nldebug("DT: parsing phrase '%s'", phrase.Identifier.c_str());
 365                 lastRead = phrase.Identifier;
 366                 CI18N::skipWhiteSpace(first, last, &phrase.Comments);
 367                 if (!CI18N::parseMarkedString('(', ')', first, last, phrase.Parameters))
 368                 {
 369                         uint32 line = countLine(doc, first);
 370                         nlwarning("DT: in '%s', line %u: Error parsing parameter list for phrase %s\n",
 371                                 filename.c_str(),
 372                                 line,
 373                                 phrase.Identifier.c_str());
 374                         return false;
 375                 }
 376                 CI18N::skipWhiteSpace(first, last, &phrase.Comments);
 377                 if (first == last || *first != '{')
 378                 {
 379                         uint32 line = countLine(doc, first);
 380                         nlwarning("DT: In '%s', line %u: Error parsing block opening '{' in phase %s\n",
 381                                 filename.c_str(),
 382                                 line,
 383                                 phrase.Identifier.c_str());
 384                         return false;
 385                 }
 386                 ++first;
 387
 388                 ucstring temp;
 389
 390                 while (first != last && *first != '}')
 391                 {
 392                         TClause clause;
 393                         // append the comment preread at previous pass
 394                         clause.Comments = temp;
 395                         temp.erase();
 396                         // parse the clauses
 397                         CI18N::skipWhiteSpace(first, last, &clause.Comments);
 398                         if (first == last)
 399                         {
 400                                 nlwarning("DT: Found end of file in non closed block for phrase %s\n", phrase.Identifier.c_str());
 401                                 return false;
 402                         }
 403
 404                         if (*first == '}')
 405                                 break;
 406
 407                         // skip the conditional expression
 408                         ucstring cond;
 409                         while (first != last && *first == '(')
 410                         {
 411                                 if (!CI18N::parseMarkedString('(', ')', first, last, cond))
 412                                 {
 413                                         uint32 line = countLine(doc, first);
 414                                         nlwarning("DT: In '%s' line %u: Error parsing conditional expression in phrase %s, clause %u\n",
 415                                                 filename.c_str(),
 416                                                 line,
 417                                                 phrase.Identifier.c_str(),
 418                                                 phrase.Clauses.size()+1);
 419                                         return false;
 420                                 }
 421
 422                                 // only prepend a space if required
 423                                 if (!clause.Conditions.empty()) clause.Conditions += " ";
 424
 425                                 clause.Conditions += "(" + cond + ")";
 426                                 CI18N::skipWhiteSpace(first, last, &clause.Comments);
 427                         }
 428
 429                         if (first == last)
 430                         {
 431                                 nlwarning("DT: in '%s': Found end of file in non closed block for phrase %s\n",
 432                                         filename.c_str(),
 433                                         phrase.Identifier.c_str());
 434                                 return false;
 435                         }
 436                         // read the idnetifier (if any)
 437                         CI18N::parseLabel(first, last, clause.Identifier);
 438                         CI18N::skipWhiteSpace(first, last, &temp);
 439                         // read the text
 440                         if (CI18N::parseMarkedString('[', ']', first, last, clause.Text))
 441                         {
 442                                 // the last read comment is for this clause.
 443                                 clause.Comments += temp;
 444                                 temp.erase();
 445                         }
 446                         else
 447                         {
 448                                 uint32 line = countLine(doc, first);
 449                                 nlwarning("DT: in '%s' line %u: Error reading text for clause %u (%s) in  phrase %s\n",
 450                                         filename.c_str(),
 451                                         line,
 452                                         phrase.Clauses.size()+1,
 453                                         clause.Identifier.c_str(),
 454                                         phrase.Identifier.c_str());
 455                                 return false;
 456
 457                         }
 458
 459                         phrase.Clauses.push_back(clause);
 460                 }
 461                 CI18N::skipWhiteSpace(first, last);
 462                 if (first == last || *first != '}')
 463                 {
 464                         uint32 line = countLine(doc, first);
 465                         nlwarning("DT: in '%s' line %u: Missing block closing tag '}' in phrase %s\n",
 466                                 filename.c_str(),
 467                                 line,
 468                                 phrase.Identifier.c_str());
 469                         return false;
 470                 }
 471                 ++first;
 472
 473                 // handle hash value.
 474                 if (forceRehash || !parseHashFromComment(phrase.Comments, phrase.HashValue))
 475                 {
 476                         // the hash is not in the comment, compute it.
 477                         phrase.HashValue = makePhraseHash(phrase);
 478                         if (forceRehash)
 479                         {
 480                                 // the has is perhaps in the comment
 481                                 ucstring::size_type pos = phrase.Comments.find(ucstring("// HASH_VALUE"));
 482                                 if (pos != ucstring::npos)
 483                                 {
 484                                         phrase.Comments = phrase.Comments.substr(0, pos);
 485                                 }
 486                         }
 487                 }
 488
 489 //              nldebug("DT : storing phrase '%s'", phrase.Identifier.c_str());
 490                 phrases.push_back(phrase);
 491         }
 492
 493         // check identifier uniqueness
 494         {
 495                 bool error = false;
 496                 set<string>     unik;
 497                 set<string>::iterator it;
 498                 for (uint i=0; i<phrases.size(); ++i)
 499                 {
 500                         it = unik.find(phrases[i].Identifier);
 501                         if (it != unik.end())
 502                         {
 503                                 nlwarning("DT: readPhraseFile : identifier '%s' exist twice", phrases[i].Identifier.c_str() );
 504                                 error = true;
 505                         }
 506                         else
 507                                 unik.insert(phrases[i].Identifier);
 508                 }
 509                 if (error)
 510                         return false;
 511         }
 512
 513         return true;
 514 }
 515 ucstring tabLines(uint nbTab, const ucstring &str)
 516 {
 517         ucstring ret;
 518         ucstring tabs;
 519
 520         for (uint i =0; i<nbTab; ++i)
 521                 tabs.push_back('\t');
 522
 523         ret = tabs;
 524         ucstring::const_iterator first(str.begin()), last(str.end());
 525         for (; first != last; ++first)
 526         {
 527                 ret += *first;
 528                 if (*first == '\n')
 529                         ret += tabs;
 530         }
 531
 532         while (ret[ret.size()-1] == '\t')
 533                 ret = ret.substr(0, ret.size()-1);
 534
 535         return ret;
 536 }
 537
 538 ucstring preparePhraseFile(const vector<TPhrase> &phrases, bool removeDiffComments)
 539 {
 540         ucstring ret;
 541         vector<TPhrase>::const_iterator first(phrases.begin()), last(phrases.end());
 542         for (; first != last; ++first)
 543         {
 544                 const TPhrase &p = *first;
 545
 546                 if (removeDiffComments)
 547                 {
 548                         string comment = p.Comments.toString();
 549                         vector<string>  lines;
 550                         explode(comment, string("\n"), lines, true);
 551
 552                         uint i;
 553                         for (i=0; i<lines.size(); ++i)
 554                         {
 555                                 if (lines[i].find("// DIFF ") != string::npos)
 556                                 {
 557                                         lines.erase(lines.begin()+i);
 558                                         --i;
 559                                 }
 560                         }
 561
 562                         comment.erase();
 563                         for (i=0; i<lines.size(); ++i)
 564                         {
 565                                 comment += lines[i] + "\n";
 566                         }
 567                         p.Comments = ucstring(comment);
 568                 }
 569                 ret += p.Comments;
 570
 571                 if (!p.Identifier.empty() || !p.Clauses.empty())
 572                 {
 573                         if (p.Comments.find(ucstring("// HASH_VALUE ")) == ucstring::npos)
 574                         {
 575                                 // add the hash value.
 576                                 ret += ucstring("// HASH_VALUE ")+CI18N::hashToString(p.HashValue) + nl;
 577                         }
 578                         ret += p.Identifier + " ("+p.Parameters + ")" + nl;
 579                         ret += '{';
 580                         ret += nl;
 581                         for (uint i=0; i<p.Clauses.size(); ++i)
 582                         {
 583                                 const TClause &c = p.Clauses[i];
 584                                 if (!c.Comments.empty())
 585                                 {
 586                                         ucstring comment = tabLines(1, c.Comments);
 587                                         ret += comment; // + '\n';
 588                                 }
 589                                 if (!c.Conditions.empty())
 590                                 {
 591                                         ucstring cond = tabLines(1, c.Conditions);
 592                                         ret += cond + nl;
 593                                 }
 594                                 ret += '\t';
 595 //                              ucstring text = CI18N::makeMarkedString('[', ']', c.Text);
 596
 597                                 ucstring text = CI18N::makeMarkedString('[', ']', c.Text);;
 598                                 ucstring text2;
 599                                 // add new line and tab after each \n tag
 600                                 ucstring::size_type pos;
 601                                 const ucstring nlTag("\\n");
 602                                 while ((pos = text.find(nlTag)) != ucstring::npos)
 603                                 {
 604                                         text2 += text.substr(0, pos+2) + nl;
 605                                         text = text.substr(pos+2);
 606                                 }
 607                                 text2 += text;//.substr(0, pos+2);
 608
 609                                 text.swap(text2);
 610
 611                                 text = tabLines(3, text);
 612                                 // remove begin tabs
 613                                 text = text.substr(3);
 614                                 ret += '\t' + c.Identifier + '\t' + text + nl + nl;
 615                         }
 616                         ret += '}';
 617                 }
 618                 ret += nl + nl;
 619         }
 620
 621         return ret;
 622 }
 623
 624 bool loadExcelSheet(const string filename, TWorksheet &worksheet, bool checkUnique)
 625 {
 626         // Yoyo: must test with CIFile because can be packed into a .bnp on client...
 627         CIFile  fp;
 628         if(!fp.open(filename))
 629         {
 630                 nldebug("DT: Can't open file [%s]\n", filename.c_str());
 631                 return true;
 632         }
 633         fp.close();
 634
 635         ucstring str;
 636         CI18N::readTextFile(filename, str, false, false, CI18N::LINE_FMT_LF);
 637
 638         if (!readExcelSheet(str, worksheet, checkUnique))
 639                 return false;
 640
 641         return true;
 642 }
 643
 644 bool readExcelSheet(const ucstring &str, TWorksheet &worksheet, bool checkUnique)
 645 {
 646         if(str.empty())
 647                 return true;
 648
 649         // copy the str to a big ucchar array => Avoid allocation / free
 650         vector<ucchar>  strArray;
 651         // append a '\0'
 652         strArray.resize(str.size()+1);
 653         strArray[strArray.size()-1]= 0;
 654         memcpy(&strArray[0], &str[0], str.size()*sizeof(ucchar));
 655
 656         // size of new line characters
 657         size_t sizeOfNl = nl.length();
 658
 659         // **** Build array of lines. just point to strArray, and fill 0 where appropriated
 660         vector<ucchar*> lines;
 661         lines.reserve(500);
 662         ucstring::size_type pos = 0;
 663         ucstring::size_type lastPos = 0;
 664         while ((pos = str.find(nl, lastPos)) != ucstring::npos)
 665         {
 666                 if (pos>lastPos)
 667                 {
 668                         strArray[pos]= 0;
 669 //                      nldebug("Found line : [%s]", ucstring(&strArray[lastPos]).toString().c_str());
 670                         lines.push_back(&strArray[lastPos]);
 671                 }
 672                 lastPos = pos + sizeOfNl;
 673         }
 674
 675         // Must add last line if no \n ending
 676         if (lastPos < str.size())
 677         {
 678                 pos= str.size();
 679                 strArray[pos]= 0;
 680 //              nldebug("Found line : [%s]", ucstring(&strArray[lastPos]).toString().c_str());
 681                 lines.push_back(&strArray[lastPos]);
 682         }
 683
 684 //      nldebug("Found %u lines", lines.size());
 685
 686         // **** Do 2 pass.1st count the cell number, then fill. => avoid reallocation
 687         uint            newColCount= 0;
 688         uint            i;
 689         for (i=0; i<lines.size(); ++i)
 690         {
 691                 uint    numCells;
 692                 numCells= 0;
 693
 694                 ucchar  *first= lines[i];
 695                 for (; *first != 0; ++first)
 696                 {
 697                         if (*first == '\t')
 698                         {
 699                                 numCells++;
 700                         }
 701                         else if (*first == '"' && first==lines[i])
 702                         {
 703                                 // read a quoted field.
 704                                 first++;
 705                                 while (*first != 0 && *first != '"' && *(first+1) != 0 && *(first+1) != '"')
 706                                 {
 707                                         first++;
 708                                         if (*first != 0 && *first == '"')
 709                                         {
 710                                                 // skip this
 711                                                 first++;
 712                                         }
 713                                 }
 714                         }
 715                 }
 716                 // last cell
 717                 numCells++;
 718
 719                 // take max cell of all lines
 720                 if (newColCount != max(newColCount, numCells))
 721                 {
 722                         newColCount = max(newColCount, numCells);
 723                         nldebug("At line %u, numCol changed to %u",
 724                                 i, newColCount);
 725                 }
 726         }
 727
 728
 729         // **** alloc / enlarge worksheet
 730         // enlarge Worksheet column size, as needed
 731         while (worksheet.ColCount < newColCount)
 732                 worksheet.insertColumn(worksheet.ColCount);
 733
 734         // enlarge Worksheet row size, as needed
 735         uint    startLine= worksheet.size();
 736         worksheet.resize(startLine + (uint)lines.size());
 737
 738
 739         // **** fill worksheet
 740         ucstring        cell;
 741         for (i=0; i<lines.size(); ++i)
 742         {
 743                 uint    numCells;
 744                 numCells= 0;
 745                 cell.erase();
 746
 747                 ucchar  *first= lines[i];
 748                 for (; *first != 0; ++first)
 749                 {
 750                         if (*first == '\t')
 751                         {
 752 //                              nldebug("Found cell [%s]", cell.toString().c_str());
 753                                 worksheet.setData(startLine + i, numCells, cell);
 754                                 numCells++;
 755                                 cell.erase();
 756                         }
 757                         else if (*first == '"' && first==lines[i])
 758                         {
 759                                 // read a quoted field.
 760                                 first++;
 761                                 while (*first != 0 && *first != '"' && *(first+1) != 0 && *(first+1) != '"')
 762                                 {
 763                                         cell += *first;
 764                                         first++;
 765                                         if (*first != 0 && *first == '"')
 766                                         {
 767                                                 // skip this
 768                                                 first++;
 769                                         }
 770                                 }
 771                         }
 772                         else
 773                         {
 774                                 cell += *first;
 775                         }
 776                 }
 777 //              nldebug("Found cell [%s]", cell.toString().c_str());
 778                 /// append last cell
 779                 worksheet.setData(startLine + i, numCells, cell);
 780                 numCells++;
 781                 nlassertex(numCells<=newColCount, ("readExcelSheet: bad row format: at line %u, the row has %u cell, max is %u", i, numCells, newColCount));
 782 //              nldebug("Found %u cells in line %u", numCells, i);
 783         }
 784
 785
 786         // **** identifier uniqueness checking.
 787         if (checkUnique)
 788         {
 789                 if (worksheet.size() > 0)
 790                 {
 791                         // look for the first non '* tagged' or 'DIFF_CMD' column
 792                         uint nameCol = 0;
 793                         while (nameCol < worksheet.ColCount && (*worksheet.getData(0, nameCol).begin() == uint16('*') || worksheet.getData(0, nameCol) == ucstring("DIFF_CMD")))
 794                                 ++nameCol;
 795
 796                         if (nameCol < worksheet.ColCount )
 797                         {
 798                                 // ok we can check unikness
 799                                 bool error = false;
 800                                 set<ucstring>   unik;
 801                                 set<ucstring>::iterator it;
 802                                 for (uint j=0; j<worksheet.size(); ++j)
 803                                 {
 804                                         it = unik.find(worksheet.getData(j, nameCol));
 805                                         if (it != unik.end())
 806                                         {
 807                                                 nlwarning("DT: readExcelSheet : identifier '%s' exist twice", worksheet.getData(j, nameCol).toString().c_str() );
 808                                                 error = true;
 809                                         }
 810                                         else
 811                                                 unik.insert(worksheet.getData(j, nameCol));
 812                                 }
 813                                 if (error)
 814                                         return false;
 815                         }
 816                 }
 817         }
 818
 819         return true;
 820 }
 821
 822 void makeHashCode(TWorksheet &sheet, bool forceRehash)
 823 {
 824         if (!sheet.Data.empty())
 825         {
 826                 TWorksheet::TRow::iterator it = find(sheet.Data[0].begin(), sheet.Data[0].end(), ucstring("*HASH_VALUE"));
 827                 if (forceRehash || it == sheet.Data[0].end())
 828                 {
 829                         // we need to generate HASH_VALUE column !
 830                         if (it == sheet.Data[0].end())
 831                         {
 832                                 sheet.insertColumn(0);
 833                                 sheet.Data[0][0] = ucstring("*HASH_VALUE");
 834                         }
 835
 836                         // Check columns
 837                         vector<bool>    columnOk;
 838                         columnOk.resize(sheet.ColCount, false);
 839                         for (uint k=1; k<sheet.ColCount; ++k)
 840                         {
 841                                 if (sheet.Data[0][k].find(ucstring("*")) != 0 && sheet.Data[0][k].find(ucstring("DIFF ")) != 0)
 842                                 {
 843                                         columnOk[k]= true;
 844                                 }
 845                         }
 846
 847                         // make hash for each line
 848                         ucstring str;
 849                         for (uint j=1; j<sheet.Data.size(); ++j)
 850                         {
 851                                 str.erase();
 852                                 for (uint k=1; k<sheet.ColCount; ++k)
 853                                 {
 854                                         if (columnOk[k])
 855                                         {
 856                                                 str += sheet.Data[j][k];
 857                                         }
 858                                 }
 859                                 uint64 hash = CI18N::makeHash(str);
 860                                 CI18N::hashToUCString(hash, sheet.Data[j][0]);
 861                         }
 862                 }
 863                 else
 864                 {
 865                         uint index = (uint)(it - sheet.Data[0].begin());
 866                         for (uint j=1; j<sheet.Data.size(); ++j)
 867                         {
 868                                 ucstring &field = sheet.Data[j][index];
 869
 870                                 if (!field.empty() && field[0] == '_')
 871                                         field = field.substr(1);
 872                         }
 873                 }
 874         }
 875 }
 876
 877 ucstring prepareExcelSheet(const TWorksheet &worksheet)
 878 {
 879         if(worksheet.Data.empty())
 880                 return ucstring();
 881
 882         // **** First pass: count approx the size
 883         uint    approxSize= 0;
 884         for (uint i=0; i<worksheet.Data.size(); ++i)
 885         {
 886                 for (uint j=0; j<worksheet.Data[i].size(); ++j)
 887                 {
 888                         approxSize+= (uint)worksheet.Data[i][j].size() + 1;
 889                 }
 890                 approxSize++;
 891         }
 892
 893         // Hash value for each column?
 894         vector<bool>    hashValue;
 895         hashValue.resize(worksheet.Data[0].size());
 896         for (uint j=0; j<worksheet.Data[0].size(); ++j)
 897         {
 898                 hashValue[j]= worksheet.Data[0][j] == ucstring("*HASH_VALUE");
 899         }
 900
 901         // **** Second pass: fill
 902         ucstring text;
 903         text.reserve(approxSize*2);
 904         for (uint i=0; i<worksheet.Data.size(); ++i)
 905         {
 906                 for (uint j=0; j<worksheet.Data[i].size(); ++j)
 907                 {
 908                         if (i > 0 && hashValue[j] && (!worksheet.Data[i][j].empty() && worksheet.Data[i][j][0] != '_'))
 909                                 text += "_";
 910                         text += worksheet.Data[i][j];
 911                         if (j != worksheet.Data[i].size()-1)
 912                                 text += '\t';
 913                 }
 914                 text += nl;
 915         }
 916
 917         return text;
 918 }
 919
 920
 921
 922
 923
 924
 925
 926 }       // namespace STRING_MANAGER
 927