?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'fixes' into main/gingo-test
[ryzomcore.git] / nel / tools / georges / georges2csv / georges2csv.cpp
blobd201229dadeef15326b707e5b6c148761a5674bd
1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2014-2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
6 //
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
11 //
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
16 //
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20 /*
21
22 Script commands:
23 OUTPUT <output file name>
24 FIELD <field name>
25 SOURCE <field name>
26 SCANFILES <extension>
27
28
29 */
30
31
32 // Misc
33 #include "nel/misc/types_nl.h"
34 #include "nel/misc/path.h"
35 #include "nel/misc/file.h"
36 #include "nel/misc/smart_ptr.h"
37 #include "nel/misc/command.h"
38 #include "nel/misc/common.h"
39 #include "nel/misc/path.h"
40 //#include "nel/memory/memory_manager.h"
41 #include "nel/misc/i18n.h"
42 #include "nel/misc/sstring.h"
43 #include "nel/misc/algo.h"
44 // Georges
45 #include "nel/georges/u_form.h"
46 #include "nel/georges/u_form_elm.h"
47 #include "nel/georges/u_form_dfn.h"
48 #include "nel/georges/u_form_loader.h"
49 #include "nel/georges/load_form.h"
50
51 // Georges, bypassing interface
52 #include "nel/georges/form.h"
53
54 // Basic C++
55 #include <iostream>
56 //#include <conio.h>
57 #include <stdio.h>
58 #include <limits>
59 //#include <io.h>
60
61 // stl
62 #include <map>
63
64 using namespace NLMISC;
65 using namespace std;
66 using namespace NLGEORGES;
67
68
69 /*
70 some handy prototypes
71 */
72 void setOutputFile(char *);
73 void addField(char *);
74 void addSource(char *);
75 void scanFiles(std::string extension);
76 void executeScriptFile(const string &);
77
78 /*
79 Some globals
80 */
81 FILE *Outf = NULL;
82
83 class CField
84 {
85 public:
86 std::string _name;
87 // bool _evaluated;
88 UFormElm::TEval _evaluated;
89 CField(const std::string &name, UFormElm::TEval eval)
90 : _name(name), _evaluated(eval)
91 { }
92 };
93 std::vector<CField> fields;
94 std::vector<std::string> files;
95
96 vector<string> inputScriptFiles;
97 vector<string> inputCsvFiles;
98 vector<string> inputSheetPaths;
99 bool inputSheetPathLoaded = false;
100 map<string, string> inputSheetPathContent;
101
102 const char *SEPARATOR = ";";
103 const char *ARRAY_SEPARATOR = "|";
104
105
106 class CDfnField
107 {
108 public:
109
110 explicit CDfnField (const std::string &name) : _isAnArray(false), _name(name)
111 {}
112
113 CDfnField (const std::string &name, const bool &isAnArray) : _isAnArray(isAnArray), _name(name)
114 {}
115
116 virtual ~CDfnField ()
117 {}
118
119 bool operator <(const CDfnField &other) const
120 {
121 return _name<other._name;
122 }
123
124 bool operator ==(const CDfnField &other) const
125 {
126 return _name==other._name;
127 }
128
129 const std::string &getName () const
130 {
131 return _name;
132 }
133
134 const bool &isAnArray () const
135 {
136 return _isAnArray;
137 }
138
139 private:
140 bool _isAnArray;
141 std::string _name;
142 };
143
144
145 /** Replace _false_ and _true_ with true and false
146 * this is used because excell force true and false in
147 * uppercase when is save the file in cvs mode.
148 */
149 void replaceTrueAndFalseTagFromCsv(vector<string> &args)
150 {
151 for (uint i=0; i<args.size(); ++i)
152 {
153 CSString str = args[i];
154
155 str = str.replace("_false_", "false");
156 str = str.replace("_true_", "true");
157
158 args[i] = str;
159 }
160 }
161
162 /** Replace false and true with _false_ and _true_
163 * this is used because excell force true and false in
164 * uppercase when is save the file in cvs mode.
165 * NB : this do the opposite jobs of the previous function
166 */
167 void replaceTrueAndFalseTagToCsv(string &arg)
168 {
169 CSString str = arg;
170
171 str.replace("false", "_false_");
172 str.replace("true", "_true_");
173
174 arg = str;
175 }
176
177
178 /*
179 Some routines for dealing with script input
180 */
181 void setOutputFile(const CSString &filename)
182 {
183 if (Outf!=NULL)
184 fclose(Outf);
185 Outf = nlfopen(filename.c_str(), "wt");
186 if (Outf == NULL)
187 {
188 fprintf(stderr, "Can't open output file '%s' ! aborting.", filename.c_str());
189 getchar();
190 exit(1);
191 }
192 fields.clear();
193 }
194
195 void addField(const CSString &name)
196 {
197 fields.push_back(CField(name, UFormElm::Eval));
198 }
199
200 void addSource(const CSString &name)
201 {
202 fields.push_back(CField(name, UFormElm::NoEval));
203 }
204
205 void buildFileVector(std::vector<std::string> &filenames, const std::string &filespec)
206 {
207 uint i,j;
208 // split up the filespec into chains
209 CSString filters = filespec;
210 filters.strip();
211 std::vector<std::string> in, out;
212
213 while (!filters.empty())
214 {
215 CSString filter = filters.strtok(" \t");
216 if (filter.empty())
217 continue;
218
219 switch (filter[0])
220 {
221 case '+':
222 in.push_back(filter.leftCrop(1)); break;
223 break;
224 case '-':
225 out.push_back(filter.leftCrop(1)); break;
226 break;
227 default:
228 fprintf(stderr,"Error in '%s' : filter must start with '+' or '-'\n",
229 filter.c_str());
230 getchar();
231 exit(1);
232 }
233 }
234
235 /* for (i=0;i<filespec.size();)
236 {
237 for (j=i;j<filespec.size() && filespec[j]!=' ' && filespec[j]!='\t';j++) {}
238 switch(filespec[i])
239 {
240 case '+':
241 in.push_back(filespec.substr(i+1,j-i-1)); break;
242 case '-':
243 out.push_back(filespec.substr(i+1,j-i-1)); break;
244 default:
245 fprintf(stderr,"Filter must start with '+' or '-'\n",&(filespec[i])); getchar(); exit(1);
246 }
247 i=j;
248 while (i<filespec.size() && (filespec[i]==' ' || filespec[i]=='\t')) i++; // skip white space
249 }
250 */
251 // use the filespec as a filter while we build the sheet file vector
252 for (i=0;i<files.size();i++)
253 {
254 bool ok=true;
255
256 // make sure the filename includes all of the include strings
257 for (j=0;j<in.size() && ok;j++)
258 {
259 if (!testWildCard(CFile::getFilename(files[i]), in[j]))
260 {
261 ok=false;
262 }
263 }
264
265 // make sure the filename includes none of the exclude strings
266 for (j=0;j<out.size() && ok;j++)
267 {
268 if (testWildCard(CFile::getFilename(files[i]), out[j]))
269 {
270 ok=false;
271 }
272 }
273
274 // if the filename matched all of the above criteria then add it to the list
275 if (ok)
276 {
277 printf("Added: %s\n",CFile::getFilename(files[i]).c_str());
278 filenames.push_back(files[i]);
279 }
280 }
281 printf("Found: %u matching files (from %u)\n",(uint)filenames.size(),(uint)files.size());
282
283 }
284
285
286 void addQuotesRoundString (std::string &valueString)
287 {
288 // add quotes round strings
289 std::string hold=valueString;
290 valueString.erase();
291 valueString='\"';
292 for (uint i=0;i<hold.size();i++)
293 {
294 if (hold[i]=='\"')
295 valueString+="\"\"";
296 else
297 valueString+=hold[i];
298 }
299 valueString+='\"';
300 }
301
302 void setErrorString (std::string &valueString, const UFormElm::TEval &evaluated, const UFormElm::TWhereIsValue &where)
303 {
304 if (evaluated==UFormElm::NoEval)
305 {
306 switch(where)
307 {
308 case UFormElm::ValueForm: valueString="ValueForm"; break;
309 case UFormElm::ValueParentForm: valueString="ValueParentForm"; break;
310 case UFormElm::ValueDefaultDfn: valueString="ValueDefaultDfn"; break;
311 case UFormElm::ValueDefaultType: valueString="ValueDefaultType"; break;
312 default: valueString="ERR";
313 }
314
315 }
316 else
317 {
318 valueString="ERR";
319 }
320
321 }
322
323
324
325 /*
326 Scanning the files ... this is the business!!
327 */
328 void scanFiles(const CSString &filespec)
329 {
330 std::vector<std::string> filenames;
331
332 buildFileVector(filenames, filespec);
333
334 // if there's no file, nothing to do
335 if (filenames.empty())
336 return;
337
338 // display the table header line
339 fprintf(Outf,"FILE");
340 for (uint i=0;i<fields.size();i++)
341 fprintf(Outf,"%s%s",SEPARATOR, fields[i]._name.c_str());
342 fprintf(Outf,"\n");
343
344 UFormLoader *formLoader = NULL;
345 NLMISC::TTime last = NLMISC::CTime::getLocalTime ();
346 NLMISC::TTime start = NLMISC::CTime::getLocalTime ();
347
348 NLMISC::CSmartPtr<UForm> form;
349
350
351 for (uint j = 0; j < filenames.size(); j++)
352 {
353 if (NLMISC::CTime::getLocalTime () > last + 5000)
354 {
355 last = NLMISC::CTime::getLocalTime ();
356 if (j>0)
357 {
358 nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/filenames.size(),j,filenames.size(), (filenames.size()-j)*(last-start)/j/1000);
359 }
360
361 }
362
363 //std::string p = NLMISC::CPath::lookup (filenames[j], false, false);
364 std::string p = filenames[j];
365 if (p.empty()) continue;
366
367 // create the georges loader if necessary
368 if (formLoader == NULL)
369 {
370 WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
371 formLoader = UFormLoader::createLoader ();
372 }
373
374 // Load the form with given sheet id
375 // form = formLoader->loadForm (sheetIds[j].toString().c_str ());
376 form = formLoader->loadForm (filenames[j].c_str ());
377 if (form)
378 {
379 // the form was found so read the true values from George
380 // std::string s;
381 fprintf(Outf,"%s",CFile::getFilenameWithoutExtension(filenames[j]).c_str());
382 for (uint i=0;i<fields.size();i++)
383 {
384 UFormElm::TWhereIsValue where;
385 UFormElm *fieldForm=NULL;
386 std::string valueString;
387
388 form->getRootNode ().getNodeByName(&fieldForm, fields[i]._name);
389
390 if (fieldForm)
391 {
392 if (fieldForm->isArray()) // if its an array
393 {
394 uint arraySize=0,arrayIndex=0;
395 fieldForm->getArraySize(arraySize);
396 while (arrayIndex<arraySize)
397 {
398 if (fieldForm->getArrayValue(valueString,arrayIndex,fields[i]._evaluated, &where))
399 ;//addQuotesRoundString (valueString);
400 else
401 setErrorString (valueString, fields[i]._evaluated, where);
402
403 arrayIndex++;
404 if (arrayIndex<arraySize) // another value in the array..
405 valueString+=ARRAY_SEPARATOR;
406 }
407
408 }
409 else
410 {
411 if (form->getRootNode ().getValueByName(valueString,fields[i]._name, fields[i]._evaluated, &where)) //fieldForm->getValue(valueString,fields[i]._evaluated))
412 ;//addQuotesRoundString (valueString);
413 else
414 setErrorString (valueString, fields[i]._evaluated, where);
415 }
416 }
417 // else // node not found.
418 // {
419 // setErrorString (valueString, fields[i]._evaluated, where);
420 // }
421
422 replaceTrueAndFalseTagToCsv(valueString);
423
424 fprintf(Outf,"%s%s", SEPARATOR, valueString.c_str());
425
426 // UFormElm::TWhereIsValue where;
427 //
428 // bool result=form->getRootNode ().getValueByName(s,fields[i]._name, fields[i]._evaluated,&where);
429 // if (!result)
430 // {
431 // if (fields[i]._evaluated)
432 // {
433 // s="ERR";
434 // }
435 // else
436 // {
437 // switch(where)
438 // {
439 // case UFormElm::ValueForm: s="ValueForm"; break;
440 // case UFormElm::ValueParentForm: s="ValueParentForm"; break;
441 // case UFormElm::ValueDefaultDfn: s="ValueDefaultDfn"; break;
442 // case UFormElm::ValueDefaultType: s="ValueDefaultType"; break;
443 // default: s="ERR";
444 // }
445 //
446 // }
447 //
448 // }
449 // else
450 // {
451 // // add quotes round strings
452 // std::string hold=s;
453 // s.erase();
454 // s='\"';
455 // for (uint i=0;i<hold.size();i++)
456 // {
457 // if (hold[i]=='\"')
458 // s+="\"\"";
459 // else
460 // s+=hold[i];
461 // }
462 // s+='\"';
463 // }
464 // fprintf(Outf,"%s%s", SEPARATOR, s);
465 }
466 fprintf(Outf,"\n");
467 }
468
469 }
470
471 // free the georges loader if necessary
472 if (formLoader != NULL)
473 {
474 UFormLoader::releaseLoader (formLoader);
475 WarningLog->removeFilter ("CFormLoader: Can't open the form file");
476 }
477
478 // housekeeping
479 // sheetIds.clear ();
480 filenames.clear ();
481
482 fields.clear();
483 }
484
485
486 //void executeScriptBuf(char *txt)
487 void executeScriptBuf(const string &text)
488 {
489 CSString buf = text;
490 CVectorSString lines;
491
492 vector<string> tmpLines;
493 NLMISC::explode(std::string(buf.c_str()), std::string("\n"), tmpLines, true);
494 lines.resize(tmpLines.size());
495 for (uint i=0; i<tmpLines.size();i++)
496 {
497 lines[i]= tmpLines[i];
498 }
499
500 for (uint i=0; i<lines.size(); ++i)
501 {
502 CSString line = lines[i];
503 line = line.strip();
504 if (line.empty() || line.find("//") == 0)
505 {
506 // comment or empty line, skip
507 continue;
508 }
509 CSString command = line.strtok(" \t");
510 line = line.strip();
511
512
513 if (command == "DFNPATH")
514 {
515 //CPath::getPathContent(args,true,false,true,files);
516 CPath::addSearchPath(line, true, false); // for the dfn files
517 }
518 else if (command == "PATH")
519 {
520 files.clear();
521 CPath::getPathContent(line, true,false,true,files);
522 CPath::addSearchPath(line, true, false); // for the dfn files
523 }
524 else if (command == "OUTPUT")
525 {
526 setOutputFile(line);
527 }
528 else if (command == "FIELD")
529 {
530 addField(line);
531 }
532 else if (command == "SOURCE")
533 {
534 addSource(line);
535 }
536 else if (command == "SCANFILES")
537 {
538 scanFiles(line);
539 }
540 else if (command == "SCRIPT")
541 {
542 executeScriptFile(line);
543 }
544 else
545 {
546 fprintf(stderr,"Unknown command: '%s' '%s'\n", command.c_str(), line.c_str());
547 }
548 }
549
550
551 }
552
553 void executeScriptFile(const string &filename)
554 {
555 ucstring temp;
556 CI18N::readTextFile(filename, temp, false, false);
557
558 if (temp.empty())
559 {
560 fprintf(stderr, "the field '%s' is empty.\n", filename.c_str());
561 return;
562 }
563 string buf = temp.toString();
564
565 executeScriptBuf(buf);
566 }
567
568 void loadSheetPath()
569 {
570 if (inputSheetPathLoaded)
571 return;
572
573 NLMISC::createDebug();
574 NLMISC::WarningLog->addNegativeFilter( "CPath::insertFileInMap" );
575
576 vector<string> files;
577 vector<string> pathsToAdd;
578 for (uint i=0; i<inputSheetPaths.size(); ++i)
579 {
580 explode( inputSheetPaths[i], std::string("*"), pathsToAdd );
581 for ( vector<string>::const_iterator ip=pathsToAdd.begin(); ip!=pathsToAdd.end(); ++ip )
582 {
583 CPath::addSearchPath( *ip, true, false );
584 CPath::getPathContent( *ip, true, false, true, files );
585 }
586 }
587
588 uint i;
589 for (i=0; i<files.size(); ++i)
590 {
591 string& filename = files[i];
592 // string& filebase = CFile::getFilenameWithoutExtension(filename);
593 const string& filebase = CFile::getFilename(filename);
594 inputSheetPathContent[filebase] = filename;
595 }
596
597 inputSheetPathLoaded = true;
598 }
599
600
601 /*
602 *
603 */
604 void fillFromDFN( UFormLoader *formLoader, set<CDfnField>& dfnFields, UFormDfn *formDfn, const string& rootName, const string& dfnFilename )
605 {
606 uint i;
607 for ( i=0; i!=formDfn->getNumEntry(); ++i )
608 {
609 string entryName, rootBase;
610 formDfn->getEntryName( i, entryName );
611 rootBase = rootName.empty() ? "" : (rootName+".");
612
613 UFormDfn::TEntryType entryType;
614 bool array;
615 formDfn->getEntryType( i, entryType, array );
616 switch ( entryType )
617 {
618 case UFormDfn::EntryVirtualDfn:
619 {
620 CSmartPtr<UFormDfn> subFormDfn = formLoader->loadFormDfn( (entryName + ".dfn").c_str() );
621 if ( ! subFormDfn )
622 nlwarning( "Can't load virtual DFN %s", entryName.c_str() );
623 else
624 fillFromDFN( formLoader, dfnFields, subFormDfn, rootBase + entryName, entryName + ".dfn" );
625 break;
626 }
627 case UFormDfn::EntryDfn:
628 {
629 UFormDfn *subFormDfn;
630 if ( formDfn->getEntryDfn( i, &subFormDfn) )
631 {
632 string filename;
633 formDfn->getEntryFilename( i, filename );
634 fillFromDFN( formLoader, dfnFields, subFormDfn, rootBase + entryName, filename ); // recurse
635 }
636 break;
637 }
638 case UFormDfn::EntryType:
639 {
640 const std::string finalName(rootBase+entryName);
641 dfnFields.insert( CDfnField(finalName, array) );
642 //nlinfo( "DFN entry: %s (in %s)", (rootBase + entryName).c_str(), dfnFilename.c_str() );
643 break;
644 }
645 }
646 }
647 }
648
649
650 /*
651 * Clear the form to reuse it (and all contents below node)
652 */
653 void clearSheet( CForm *form, UFormElm* node )
654 {
655 ((CFormElm*)node)->clean();
656 form->clean();
657 }
658
659
660 /*
661 * - Remove CSV carriage returns.
662 * - Ensure there is no non-ascii char (such as Excel's special blank crap), set them to ' '.
663 */
664 void eraseCarriageReturnsAndMakeBlankNonAsciiChars( string& s )
665 {
666 const char CR = '\n';
667 string::size_type p = s.find( CR );
668 while ( (p=s.find( CR )) != string::npos )
669 s.erase( p, 1 );
670 for ( p=0; p!=s.size(); ++p )
671 {
672 uint8& c = (uint8&)s[p]; // ensure the test is unsigned
673 if ( c > 127 )
674 {
675 //nldebug( "Blanking bad char %u in '%s'", c, s.c_str() );
676 s[p] = ' ';
677 }
678 }
679 }
680
681
682 string OutputPath;
683
684
685 /*
686 * CSV -> Georges
687 */
688 void convertCsvFile( const string &file, bool generate, const string& sheetType )
689 {
690 const uint BUFFER_SIZE = 16*1024;
691 char lineBuffer[BUFFER_SIZE];
692
693 vector<string> fields;
694 vector<string> args;
695
696 FILE *s = nlfopen(file, "r");
697
698 if (s == NULL)
699 {
700 fprintf(stderr, "Can't find file %s to convert\n", file.c_str());
701 return;
702 }
703
704 if (!fgets(lineBuffer, BUFFER_SIZE, s))
705 {
706 nlwarning("fgets() failed");
707 return;
708 }
709
710 loadSheetPath();
711
712 UFormLoader *formLoader = UFormLoader::createLoader ();
713 NLMISC::CSmartPtr<CForm> form;
714 NLMISC::CSmartPtr<UFormDfn> formDfn;
715
716 explode(std::string(lineBuffer), std::string(SEPARATOR), fields);
717
718 vector<bool> activeFields( fields.size(), true );
719
720 // Load DFN (generation only)
721 set<CDfnField> dfnFields;
722 if ( generate )
723 {
724 formDfn = formLoader->loadFormDfn( (sheetType + ".dfn").c_str() );
725 if ( ! formDfn )
726 nlerror( "Can't find DFN for %s", sheetType.c_str() );
727 fillFromDFN( formLoader, dfnFields, formDfn, "", sheetType );
728
729 // Display missing fields and check fields against DFN
730 uint i;
731 for ( i=1; i!=fields.size(); ++i )
732 {
733 eraseCarriageReturnsAndMakeBlankNonAsciiChars( fields[i] );
734 if ( fields[i].empty() )
735 {
736 nlinfo( "Skipping field #%u (empty)", i );
737 activeFields[i] = false;
738 }
739 else if ( nlstricmp( fields[i], "parent" ) == 0 )
740 {
741 fields[i] = toLowerAscii( fields[i] );
742 }
743 else
744 {
745 set<CDfnField>::iterator ist = dfnFields.find( CDfnField(fields[i]) );
746 if ( ist == dfnFields.end() )
747 {
748 nlinfo( "Skipping field #%u (%s, not found in %s DFN)", i, fields[i].c_str(), sheetType.c_str() );
749 activeFields[i] = false;
750 }
751 }
752 }
753 for ( i=1; i!=fields.size(); ++i )
754 {
755 if ( activeFields[i] )
756 nlinfo( "Selected field: %s", fields[i].c_str() );
757 }
758 }
759
760 string addExtension = "." + sheetType;
761 uint dirmapLetterIndex = std::numeric_limits<uint>::max();
762 bool dirmapLetterBackward = false;
763 vector<string> dirmapDirs;
764 string dirmapSheetCode;
765 bool WriteEmptyProperties = false, WriteSheetsToDisk = true;
766 bool ForceInsertParents = false;
767
768 if ( generate )
769 {
770 // Get the directory mapping
771 try
772 {
773 CConfigFile dirmapcfg;
774 dirmapcfg.load( sheetType + "_dirmap.cfg" );
775
776 if ( OutputPath.empty() )
777 {
778 CConfigFile::CVar *path = dirmapcfg.getVarPtr( "OutputPath" );
779 if ( path )
780 OutputPath = path->asString();
781 if ( ! OutputPath.empty() )
782 {
783 if ( OutputPath[OutputPath.size()-1] != '/' )
784 OutputPath += '/';
785 else if ( ! CFile::isDirectory( OutputPath ) )
786 nlwarning( "Output path does not exist" );
787 }
788 }
789
790 CConfigFile::CVar *letterIndex1 = dirmapcfg.getVarPtr( "LetterIndex" );
791 if ( letterIndex1 && letterIndex1->asInt() > 0 )
792 {
793 dirmapLetterIndex = letterIndex1->asInt() - 1;
794
795 CConfigFile::CVar *letterWay = dirmapcfg.getVarPtr( "LetterWay" );
796 dirmapLetterBackward = (letterWay && (letterWay->asInt() == 1));
797
798 CConfigFile::CVar dirs = dirmapcfg.getVar( "Directories" );
799 for ( uint idm=0; idm!=dirs.size(); ++idm )
800 {
801 dirmapDirs.push_back( dirs.asString( idm ) );
802 nlinfo( "Directory: %s", dirmapDirs.back().c_str() );
803 if ( ! CFile::isExists( OutputPath + dirmapDirs.back() ) )
804 {
805 CFile::createDirectory( OutputPath + dirmapDirs.back() );
806 }
807 else
808 {
809 if ( ! CFile::isDirectory( OutputPath + dirmapDirs.back() ) )
810 {
811 nlwarning( "Already existing but not a directory!" );
812 }
813 }
814 }
815
816 nlinfo( "Mapping letter #%u (%s) of sheet name to directory", dirmapLetterIndex + 1, dirmapLetterBackward?"backward":"forward" );
817 }
818
819 CConfigFile::CVar *sheetCode = dirmapcfg.getVarPtr( "AddSheetCode" );
820 if ( sheetCode )
821 dirmapSheetCode = sheetCode->asString();
822 nlinfo( "Sheet code: %s", dirmapSheetCode.c_str() );
823
824 if ( ! dirmapLetterBackward )
825 dirmapLetterIndex += (uint)dirmapSheetCode.size();
826
827 CConfigFile::CVar *wep = dirmapcfg.getVarPtr( "WriteEmptyProperties" );
828 if ( wep )
829 WriteEmptyProperties = (wep->asInt() == 1);
830 nlinfo( "Write empty properties mode: %s", WriteEmptyProperties ? "ON" : "OFF" );
831
832 CConfigFile::CVar *wstd = dirmapcfg.getVarPtr( "WriteSheetsToDisk" );
833 if ( wstd )
834 WriteSheetsToDisk = (wstd->asInt() == 1);
835 nlinfo( "Write sheets to disk mode: %s", WriteSheetsToDisk ? "ON" : "OFF" );
836
837 CConfigFile::CVar *fiparents = dirmapcfg.getVarPtr( "ForceInsertParents" );
838 if ( fiparents )
839 ForceInsertParents = (fiparents->asInt() == 1);
840 nlinfo( "Force insert parents mode: %s", ForceInsertParents ? "ON" : "OFF" );
841 }
842 catch (const EConfigFile &e)
843 {
844 nlwarning( "Problem in directory mapping: %s", e.what() );
845 }
846
847
848 nlinfo( "Using output path: %s", OutputPath.c_str() );
849 nlinfo( "Press a key to generate *.%s", sheetType.c_str() );
850 getchar();
851 nlinfo( "Generating...." );
852
853 }
854 else
855 nlinfo("Updating modifications (only modified fields are updated)");
856
857 set<string> newSheets;
858 uint nbNewSheets = 0, nbModifiedSheets = 0, nbUnchangedSheets = 0, nbWritten = 0;
859 while (!feof(s))
860 {
861 lineBuffer[0] = '\0';
862 if (!fgets(lineBuffer, BUFFER_SIZE, s))
863 {
864 nlwarning("fgets() failed");
865 break;
866 }
867
868 explode(std::string(lineBuffer), std::string(SEPARATOR), args);
869
870 if (args.size() < 1)
871 continue;
872
873 eraseCarriageReturnsAndMakeBlankNonAsciiChars( args[0] );
874 replaceTrueAndFalseTagFromCsv(args);
875
876 // Skip empty lines
877 if ( args[0].empty() || (args[0] == string(".")+sheetType) )
878 continue;
879
880 //nldebug( "%s: %u", args[0].c_str(), args.size() );
881 string filebase = dirmapSheetCode+args[0]; /*+"."+sheetType;*/
882 if (filebase.find("."+sheetType) == string::npos)
883 {
884 filebase += "." + sheetType;
885 }
886 filebase = toLowerAscii(filebase);
887 string filename, dirbase;
888 bool isNewSheet=true;
889
890 // Locate existing sheet
891 // map<string, string>::iterator it = inputSheetPathContent.find( CFile::getFilenameWithoutExtension( filebase ) );
892 map<string, string>::iterator it = inputSheetPathContent.find( CFile::getFilename( filebase ) );
893
894 if (it == inputSheetPathContent.end())
895 {
896 // Not found
897 if ( ! generate )
898 {
899 if ( ! filebase.empty() )
900 {
901 nlwarning( "Sheet %s not found", filebase.c_str( ));
902 continue;
903 }
904 }
905 else
906 {
907 // Load template sheet
908 filename = toLowerAscii(filebase);
909 form = (CForm*)formLoader->loadForm( (string("_empty.") + sheetType).c_str() );
910 if (form == NULL)
911 {
912 nlerror( "Can't load sheet _empty.%s", sheetType.c_str() );
913 }
914
915 // Deduce directory from sheet name
916 if ( dirmapLetterIndex != std::numeric_limits<uint>::max() )
917 {
918 if ( dirmapLetterIndex < filebase.size() )
919 {
920 uint letterIndex;
921 char c;
922 if ( dirmapLetterBackward )
923 letterIndex = (uint)(filebase.size() - 1 - (CFile::getExtension( filebase ).size()+1)) - dirmapLetterIndex;
924 else
925 letterIndex = dirmapLetterIndex;
926 c = tolower( filebase[letterIndex] );
927 vector<string>::const_iterator idm;
928 for ( idm=dirmapDirs.begin(); idm!=dirmapDirs.end(); ++idm )
929 {
930 if ( (! (*idm).empty()) && (tolower((*idm)[0]) == c) )
931 {
932 dirbase = (*idm) + "/";
933 break;
934 }
935 }
936 if ( idm==dirmapDirs.end() )
937 {
938 nlinfo( "Directory mapping not found for %s (index %u)", filebase.c_str(), letterIndex );
939 dirbase.clear(); // put into root
940 }
941 }
942 else
943 {
944 nlerror( "Can't map directory with letter #%u, greater than size of %s + code", dirmapLetterIndex, filebase.c_str() );
945 }
946 }
947
948 nlinfo( "New sheet: %s", filebase.c_str() );
949 ++nbNewSheets;
950 if ( ! newSheets.insert( filebase ).second )
951 nlwarning( "Found duplicate sheet: %s", filebase.c_str() );
952 isNewSheet = true;
953 }
954 }
955 else // an existing sheet was found
956 {
957
958 // Load sheet (skip if failed)
959 dirbase.clear();
960 filename = (*it).second; // whole path
961 form = (CForm*)formLoader->loadForm( filename.c_str() );
962 if (form == NULL)
963 {
964 nlwarning( "Can't load sheet %s", filename.c_str() );
965 continue;
966 }
967
968 isNewSheet = false;
969 }
970
971 const UFormElm &rootForm=form->getRootNode();
972 bool displayed = false;
973 bool isModified = false;
974 uint i;
975 for ( i=1; i<args.size ()
976 && i<fields.size (); ++i )
977 {
978 const string &var = fields[i];
979 string &val = args[i];
980
981 eraseCarriageReturnsAndMakeBlankNonAsciiChars( val );
982
983 // Skip column with inactive field (empty or not in DFN)
984 if ( (! activeFields[i]) )
985 continue;
986
987 // Skip setting of empty cell except if required
988 if ( (! WriteEmptyProperties) && val.empty() )
989 continue;
990
991 // Special case for parent sheet
992 if (var == "parent") // already case-lowered
993 {
994 vector<string> parentVals;
995 explode( val, std::string(ARRAY_SEPARATOR), parentVals );
996 if ( (parentVals.size() == 1) && (parentVals[0].empty()) )
997 parentVals.clear();
998
999 if ( (isNewSheet || ForceInsertParents) && (! parentVals.empty()) )
1000 {
1001 // This is slow. Opti: insertParent() should have an option to do it without loading the form
1002 // parent have same type that this object (postulat).
1003 uint nbinsertedparents=0;
1004
1005 for ( uint p=0; p!=parentVals.size(); ++p )
1006 {
1007 string localExtension=(parentVals[p].find(addExtension)==string::npos)?addExtension:"";
1008 string parentName=parentVals[p]+localExtension;
1009
1010 CSmartPtr<CForm> parentForm = (CForm*)formLoader->loadForm(CFile::getFilename(parentName.c_str()).c_str());
1011 if ( ! parentForm )
1012 {
1013 nlwarning( "Can't load parent form %s", parentName.c_str() );
1014 }
1015 else
1016 {
1017 form->insertParent( p, parentName.c_str(), parentForm );
1018 isModified=true;
1019 displayed = true;
1020 nbinsertedparents++;
1021 }
1022
1023 }
1024 nlinfo( "Inserted %u parent(s)", nbinsertedparents );
1025 }
1026 // NOTE: Changing the parent is not currently implemented!
1027 continue;
1028 }
1029
1030 const UFormElm *fieldForm=NULL;
1031
1032 if (rootForm.getNodeByName(&fieldForm, var))
1033 {
1034 UFormDfn *dfnForm=const_cast<UFormElm&>(rootForm).getStructDfn();
1035 nlassert(dfnForm);
1036
1037 vector<string> memberVals;
1038 explode( val, std::string(ARRAY_SEPARATOR), memberVals );
1039 uint32 memberIndex=0;
1040
1041 while (memberIndex<memberVals.size())
1042 {
1043 const uint currentMemberIndex=memberIndex;
1044 std::string memberVal=memberVals[memberIndex];
1045 memberIndex++;
1046
1047 if (!memberVal.empty())
1048 {
1049 if (memberVal[0] == '"')
1050 memberVal.erase(0, 1);
1051 if (memberVal.size()>0 && memberVal[memberVal.size()-1] == '"')
1052 memberVal.resize(memberVal.size()-1);
1053
1054 if (memberVal == "ValueForm" ||
1055 memberVal == "ValueParentForm" ||
1056 memberVal == "ValueDefaultDfn" ||
1057 memberVal == "ValueDefaultType" ||
1058 memberVal == "ERR")
1059 continue;
1060 }
1061
1062
1063 // nlassert(fieldDfn);
1064 // virtual bool getEntryFilenameExt (uint entry, std::string &name) const = 0;
1065 // virtual bool getEntryFilename (uint entry, std::string &name) const = 0;
1066 if (dfnForm)
1067 {
1068 string fileName;
1069 string fileNameExt;
1070 bool toto=false;
1071 static string filenameTyp("filename.typ");
1072 string extension;
1073
1074 uint fieldIndex;
1075 if (dfnForm->getEntryIndexByName (fieldIndex, var)) // field exists.
1076 {
1077 dfnForm->getEntryFilename(fieldIndex,fileName);
1078 if (fileName==filenameTyp)
1079 {
1080 dfnForm->getEntryFilenameExt(fieldIndex,fileNameExt);
1081 if ( !fileNameExt.empty()
1082 && fileNameExt!="*.*")
1083 {
1084 string::size_type index=fileNameExt.find(".");
1085 if (index==string::npos) // not found.
1086 {
1087 extension=fileNameExt;
1088 }
1089 else
1090 {
1091 extension=fileNameExt.substr(index+1);
1092 }
1093
1094 if (memberVal.find(extension)==string::npos) // extension not found.
1095 {
1096 memberVal=NLMISC::toString("%s.%s",memberVal.c_str(),extension.c_str());
1097 }
1098
1099 }
1100
1101 }
1102
1103 }
1104
1105 }
1106
1107
1108 if (dfnForm->isAnArrayEntryByName(var))
1109 {
1110 if ( !isNewSheet
1111 && fieldForm!=NULL)
1112 {
1113 uint arraySize;
1114 const UFormElm *arrayNode = NULL;
1115 if (fieldForm->isArray()
1116 && fieldForm->getArraySize(arraySize) && arraySize == memberVals.size())
1117 {
1118 string test;
1119 if ( fieldForm->getArrayValue(test, currentMemberIndex)
1120 && test==memberVal )
1121 {
1122 continue;
1123 }
1124 }
1125 }
1126 //nldebug( "%s: %s '%s'", args[0].c_str(), var.c_str(), memberVal.c_str() );
1127 // need to put the value at the correct index.
1128 const std::string fieldName=NLMISC::toString("%s[%u]", var.c_str(), currentMemberIndex).c_str();
1129 const_cast<UFormElm&>(rootForm).setValueByName(memberVal, fieldName);
1130 isModified=true;
1131 displayed = true;
1132 }
1133 else
1134 {
1135 if (!isNewSheet)
1136 {
1137 string test;
1138 if ( rootForm.getValueByName(test, var)
1139 && test==memberVal )
1140 {
1141 continue;
1142 }
1143
1144 }
1145 //nldebug( "%s: %s '%s'", args[0].c_str(), var.c_str(), memberVal.c_str() );
1146 const_cast<UFormElm&>(rootForm).setValueByName(memberVal, var);
1147 isModified=true;
1148 displayed = true;
1149 }
1150
1151 if (!isNewSheet)
1152 {
1153 isModified = true;
1154 if (!displayed)
1155 nlinfo("in %s:", filename.c_str());
1156 displayed = true;
1157 nlinfo("%s = %s", var.c_str(), memberVal.c_str());
1158 }
1159
1160 }
1161
1162 }
1163 else // field Node not found :\ (bad)
1164 {
1165 }
1166
1167 }
1168
1169 if ( ! isNewSheet )
1170 {
1171 if ( isModified )
1172 ++nbModifiedSheets;
1173 else
1174 ++nbUnchangedSheets;
1175 }
1176
1177 // Write sheet
1178 if ( isNewSheet || displayed )
1179 {
1180 if ( WriteSheetsToDisk )
1181 {
1182 ++nbWritten;
1183 string path = isNewSheet ? OutputPath : "";
1184 string ext = (filename.find( addExtension ) == string::npos) ? addExtension : "";
1185 string absoluteFileName=path + dirbase + filename + ext;
1186
1187 // nlinfo("opening: %s", absoluteFileName.c_str() );
1188 COFile output(absoluteFileName);
1189 if (!output.isOpen())
1190 {
1191 nlinfo("creating path: %s", (path + dirbase).c_str() );
1192 NLMISC::CFile::createDirectory(path + dirbase);
1193 }
1194
1195 // nlinfo("opening2: %s", absoluteFileName.c_str() );
1196 output.open (absoluteFileName);
1197
1198 if (!output.isOpen())
1199 {
1200 nlinfo("ERROR! cannot create file path: %s", absoluteFileName.c_str() );
1201 }
1202 else
1203 {
1204 form->write(output);
1205 output.close();
1206
1207 if (!CPath::exists(filename + ext))
1208 CPath::addSearchFile(absoluteFileName);
1209 }
1210
1211 }
1212 clearSheet( form, &form->getRootNode() );
1213 }
1214
1215 }
1216 nlinfo( "%u sheets processed (%u new, %u modified, %u unchanged - %u written)", nbNewSheets+nbModifiedSheets+nbUnchangedSheets, nbNewSheets, nbModifiedSheets, nbUnchangedSheets, nbWritten );
1217 UFormLoader::releaseLoader (formLoader);
1218 }
1219
1220 //
1221 void usage(char *argv0, FILE *out)
1222 {
1223 fprintf(out, "\n");
1224 fprintf(out, "Syntax: %s [-p <sheet path>] [-s <field_separator>] [-g <sheet type>] [-o <output path>] [<script file name> | <csv file name>]", argv0);
1225 fprintf(out, "(-g = generate sheet files, needs template sheet _empty.<sheet type> and <sheet type>_dirmap.cfg in the current folder");
1226 fprintf(out, "\n");
1227 fprintf(out, "Script commands:\n");
1228 fprintf(out, "\tDFNPATH\t\t<search path for george dfn files>\n");
1229 fprintf(out, "\tPATH\t\t<search path for files to scan>\n");
1230 fprintf(out, "\tOUTPUT\t\t<output file>\n");
1231 fprintf(out, "\tFIELD\t\t<field in george file>\n");
1232 fprintf(out, "\tSOURCE\t\t<field in george file>\n");
1233 fprintf(out, "\tSCANFILES\t[+<text>|-<text>[...]]\n");
1234 fprintf(out, "\tSCRIPT\t\t<script file to execute>\n");
1235 fprintf(out, "\n");
1236 }
1237
1238 int main(int argc, char* argv[])
1239 {
1240 bool generate = false;
1241 string sheetType;
1242
1243 // parse command line
1244 uint i;
1245 for (i=1; (sint)i<argc; i++)
1246 {
1247 const char *arg = argv[i];
1248 if (arg[0] == '-')
1249 {
1250 switch (arg[1])
1251 {
1252 case 'p':
1253 ++i;
1254 if ((sint)i == argc)
1255 {
1256 fprintf(stderr, "Missing <sheet path> after -p option\n");
1257 usage(argv[0], stderr);
1258 exit(0);
1259 }
1260 inputSheetPaths.push_back(argv[i]);
1261 break;
1262 case 's':
1263 ++i;
1264 if ((sint)i == argc)
1265 {
1266 fprintf(stderr, "Missing <field_separator> after -s option\n");
1267 usage(argv[0], stderr);
1268 exit(0);
1269 }
1270 SEPARATOR = argv[i];
1271 break;
1272 case 'g':
1273 ++i;
1274 if ((sint)i == argc)
1275 {
1276 fprintf(stderr, "Missing <sheetType> after -g option\n");
1277 usage(argv[0], stderr);
1278 exit(0);
1279 }
1280 generate = true;
1281 sheetType = string(argv[i]);
1282 break;
1283 case 'o':
1284 ++i;
1285 if ((sint)i == argc)
1286 {
1287 fprintf(stderr, "Missing <output path> after -o option\n");
1288 usage(argv[0], stderr);
1289 exit(0);
1290 }
1291 OutputPath = string(argv[i]);
1292 break;
1293 default:
1294 fprintf(stderr, "Unrecognized option '%c'\n", arg[1]);
1295 usage(argv[0], stderr);
1296 exit(0);
1297 break;
1298 }
1299 }
1300 else
1301 {
1302 if (CFile::getExtension(arg) == "csv")
1303 {
1304 inputCsvFiles.push_back(arg);
1305 }
1306 else
1307 {
1308 inputScriptFiles.push_back(arg);
1309 }
1310 }
1311 }
1312
1313 if (inputScriptFiles.empty() && inputCsvFiles.empty())
1314 {
1315 fprintf(stderr, "Missing input script file or csv file\n");
1316 usage(argv[0], stderr);
1317 exit(0);
1318 }
1319
1320
1321
1322 for (i=0; i<inputScriptFiles.size(); ++i)
1323 executeScriptFile(inputScriptFiles[i]);
1324
1325 for (i=0; i<inputCsvFiles.size(); ++i)
1326 convertCsvFile(inputCsvFiles[i], generate, sheetType);
1327
1328 fprintf(stderr,"\nDone.\n");
1329 getchar();
1330 return 0;
1331 }
1332