?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Added aqua_speed for rite geo 50 tryker
[ryzomcore.git] / nel / tools / misc / extract_filename / extract_filename.cpp
blob3dbf05b8d84ed6a87e423491e26777f142233102
1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
6 //
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
11 //
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
16 //
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20 #include "nel/misc/file.h"
21 #include "nel/misc/config_file.h"
22
23 using namespace std;
24 using namespace NLMISC;
25
26 #ifdef NL_DEBUG
27 #define INFO nlinfo
28 #else // NL_DEBUG
29 # if defined(NL_COMP_VC) && NL_COMP_VC_VERSION >= 71
30 # define INFO __noop
31 # else
32 # define INFO 0&&
33 # endif
34 #endif // NL_DEBUG
35
36 bool readTheFile (const char *filename, vector<char> &fileArray)
37 {
38 // Open the file
39 CIFile file;
40 if (file.open (filename))
41 {
42 // Go to the end
43 file.seek (0, NLMISC::IStream::end);
44
45 // Get file size
46 sint32 size = file.getPos ();
47
48 // Resize the array
49 fileArray.resize (size);
50
51 // Go to the begin
52 file.seek (0, NLMISC::IStream::begin);
53
54 if (size)
55 {
56 // Read the file
57 NLMISC::IStream *stream = &file;
58 stream->serialBuffer ((uint8*)&fileArray[0], size);
59 }
60
61 return true;
62 }
63 else
64 {
65 // Error, can't open the file
66 nlwarning ("Can't open the file %s for reading.", filename);
67
68 return false;
69 }
70 }
71
72 inline bool isASCII (uint8 c)
73 {
74 return ( (c>=0x20 /*&& c<=0x7E*/) || (c == 0x09) || (c == 0x0A) || (c == 0x0D) );
75 }
76
77 bool isASCII(vector<char> &fileArray)
78 {
79 // Array size
80 uint size = (uint)fileArray.size();
81
82 if (size)
83 {
84 // Array pointer
85 const char *arrayPointer = &fileArray[0];
86
87 // Line count
88 uint line = 0;
89
90 for (uint c=0; c<size; c++)
91 {
92 if (!isASCII (arrayPointer[c]))
93 {
94 INFO ("Binary character: 0x%02x (line %d)", (uint)(uint8)arrayPointer[c], line);
95 return false;
96 }
97 if (arrayPointer[c] == '\n')
98 line++;
99 }
100 }
101
102 return true;
103 }
104
105 inline bool isStringChar (char c)
106 {
107 return ( ( (c>='a') && (c<='z') ) || ( (c>='A') && (c<='Z') ) || ( (c>='0') && (c<='9') ) || (c=='-') || (c=='_') || (c=='.') );
108 }
109
110 void removeBeginEndSpaces (string &str)
111 {
112 // Remove begin space
113 sint index=0;
114 while ( (index<(sint)str.length ()) && (str[index]==' ') )
115 index++;
116 if (index != 0)
117 str = str.substr (index, str.length ()-index);
118
119 // Remove end space
120 index=(sint)str.length ()-1;
121 while ( (index>0) && (str[index]==' ') )
122 index--;
123 str.resize (index+1);
124 }
125
126 void removeChar (string &str, char ch)
127 {
128 for (uint c=0; c<str.size (); c++)
129 {
130 if (str[c] == ch)
131 str.erase (c, 1);
132 }
133 }
134
135 void removeDirectory (string &str)
136 {
137 // Remove begin space
138 string::size_type pos = str.rfind ('\\');
139 string::size_type pos2 = str.rfind ('/');
140
141 if (pos == string::npos)
142 pos = pos2;
143
144 if (pos != string::npos)
145 str = str.substr (pos+1, str.size());
146 }
147
148 bool filterExtension (const char *str, const vector<string> &extensions)
149 {
150 // String size
151 uint size = (uint)strlen (str);
152
153 // For each filter
154 for (uint i=0; i<extensions.size(); i++)
155 {
156 if ( (str+(size - extensions[i].size())) == extensions[i])
157 return true;
158 }
159
160 // Not found
161 return false;
162 }
163
164 bool validateFilename (const char *str)
165 {
166 // Look for space
167 return (strchr (str, ' ') == NULL);
168 }
169
170 const char *getExtension (const char *filename)
171 {
172 return strrchr (filename, '.');
173 }
174
175 void removeExtension (string &filename)
176 {
177 string::size_type index = filename.rfind ('.');
178 filename.resize (index);
179 }
180
181 void extractStringsFromBinary (const vector<char> &fileArray, set<string> &filenameStrings, const vector<string> &extensions)
182 {
183 // Array size
184 uint size = (uint)fileArray.size();
185
186 // Array pointer
187 const char *arrayPointer = &fileArray[0];
188
189 // run through the data buffer looking for plausible looking strings
190 uint i,j;
191 for (i=0; (j=i+sizeof(uint))<size; i++)
192 {
193 // if we're pointing at a string the first 4 bytes ar the string length
194 uint len=*(uint *)(arrayPointer+i);
195
196 // if the string length could be valid
197 if (len>0 && len+i<=size)
198 {
199 uint k;
200 for (k=j; k<len+j && isASCII (arrayPointer[k]);k++);
201 if (k==len+j)
202 {
203 // build a string
204 string str;
205
206 // Resize the string
207 str.resize (len);
208
209 // Copy the string
210 for (k=0; k<len; k++)
211 str[k] = arrayPointer[j+k];
212
213 // Remove space
214 removeBeginEndSpaces (str);
215
216 // Not empty ?
217 if (str != "")
218 {
219 // Lower case
220 str = toLowerAscii (str);
221
222 // Filter extensions
223 if (filterExtension (str.c_str(), extensions))
224 {
225 // Remove directory
226 removeDirectory (str);
227
228 // Validate filename
229 if (validateFilename (str.c_str()))
230 {
231 // Add a string
232 filenameStrings.insert (str);
233
234 // Information
235 INFO ("Binary filename extracted: \"%s\" (0x%08x)", str.c_str(), j);
236 }
237 else
238 {
239 INFO ("Invalid filename: \"%s\" (0x%08x)", str.c_str(), j);
240 }
241 }
242 }
243 }
244 }
245 }
246 }
247
248 void extractStringsFromASCII (const vector<char> &fileArray, set<string> &filenameStrings, const vector<string> &extensions)
249 {
250 // Array size
251 uint size = (uint)fileArray.size();
252 if (size)
253 {
254 // Array pointer
255 const char *arrayPointer = &fileArray[0];
256
257 // Temp string
258 string temp;
259
260 // Begin of a valid string
261 const char *begin = arrayPointer;
262 while ((begin<arrayPointer) && (!isStringChar (*begin)))
263 begin++;
264 const char *end = begin;
265 while (end<arrayPointer+size)
266 {
267 // End char is ok ?
268 if (isStringChar (*end))
269 {
270 end++;
271 }
272 else
273 {
274 // Found a string ?
275 if (end != begin)
276 {
277 // String size
278 uint size = (uint)(end-begin);
279 temp.resize (size);
280
281 // Copy the string
282 for (uint c=0; c<size; c++)
283 temp[c] = begin[c];
284
285 // Lower case
286 temp = toLowerAscii (temp);
287
288 // Filter extensions
289 if (filterExtension (temp.c_str(), extensions))
290 {
291 // Remove directory
292 removeDirectory (temp);
293
294 // Validate filename
295 if (validateFilename (temp.c_str()))
296 {
297 // Add a string
298 filenameStrings.insert (temp);
299
300 // Information
301 INFO ("ASCII filename extracted: \"%s\"", temp.c_str());
302 }
303 else
304 {
305 INFO ("Invalid filename: \"%s\"", temp.c_str());
306 }
307 }
308 }
309 end++;
310 begin = end;
311 }
312 }
313 }
314 }
315
316 bool loadConfigFiles (const char *ext, const char *input_files, const char *available_files, vector<string> &extensions, set<string> &inputFiles, map<string, string> &availableFiles)
317 {
318 // Get a string
319 string temp;
320 string temp2;
321
322 // Extensions files
323 FILE *file = fopen (ext, "r");
324 if (file)
325 {
326 bool cont = true;
327 char name[512];
328 while (fgets (name, 512, file))
329 {
330 // To string and lower
331 temp = toLowerAscii (string(name));
332
333 // Remove return
334 removeChar (temp, '\n');
335
336 // Valid extension ?
337 if (temp.size() && temp[0] == '.')
338 // Add the extension
339 extensions.push_back (temp);
340 else
341 {
342 nlwarning ("ERROR extension %s must begin with a '.' character.", temp.c_str());
343 cont = false;
344 break;
345 }
346 }
347
348 // Close
349 fclose (file);
350 if (cont)
351 {
352 // Input files
353 file = fopen (input_files, "r");
354 if (file)
355 {
356 char name[512];
357 while (fgets (name, 512, file))
358 {
359 // To string
360 temp = name;
361
362 // Remove return
363 removeChar (temp, '\n');
364
365 // Add the extension
366 inputFiles.insert (temp);
367 }
368
369 // Close
370 fclose (file);
371
372 // Available files
373 file = fopen (available_files, "r");
374 if (file)
375 {
376 char name[512];
377 while (fgets (name, 512, file))
378 {
379 // To lower
380 temp = toLowerAscii (string(name));
381 temp2 = toLowerAscii (string(name));
382
383 // Remove space
384 removeBeginEndSpaces (temp);
385 removeBeginEndSpaces (temp2);
386
387 // Remove return
388 removeChar (temp, '\n');
389 removeChar (temp2, '\n');
390
391 // Remove directory
392 removeDirectory (temp);
393
394 // Good extension
395 if (filterExtension (temp.c_str (), extensions))
396 {
397 if (validateFilename (temp.c_str ()))
398 {
399 // Add the extension
400 if (!availableFiles.insert (map<string, string>::value_type (temp, temp2)).second)
401 {
402 fprintf (stderr, "DUBLING: %s %s\n", temp.c_str (), temp2.c_str());
403 }
404 }
405 else
406 {
407 fprintf (stderr, "INVALIDE NAME: %s\n", temp.c_str ());
408 }
409 }
410 else
411 {
412 fprintf (stderr, "INVALIDE EXT: %s\n", temp.c_str ());
413 }
414 }
415
416 // Close
417 fclose (file);
418
419 // Ok
420 return true;
421 }
422 else
423 {
424 nlwarning ("ERROR can't load available files %s", ext);
425 }
426 }
427 else
428 {
429 nlwarning ("ERROR can't load input files %s", ext);
430 }
431 }
432 }
433 else
434 {
435 nlwarning ("ERROR can't load extension file %s", ext);
436 }
437
438 return false;
439 }
440
441
442 #define BAR_LENGTH 21
443
444 const char *progressbar[BAR_LENGTH]=
445 {
446 "[ ]",
447 "[. ]",
448 "[.. ]",
449 "[... ]",
450 "[.... ]",
451 "[..... ]",
452 "[...... ]",
453 "[....... ]",
454 "[........ ]",
455 "[......... ]",
456 "[.......... ]",
457 "[........... ]",
458 "[............ ]",
459 "[............. ]",
460 "[.............. ]",
461 "[............... ]",
462 "[................ ]",
463 "[................. ]",
464 "[.................. ]",
465 "[................... ]",
466 "[....................]"
467 };
468
469 void progress (const char *message, float progress)
470 {
471 // Progress bar
472 char msg[512];
473 uint pgId= (uint)(progress*(float)BAR_LENGTH);
474 pgId= min(pgId, (uint)(BAR_LENGTH-1));
475 sprintf (msg, "\r%s %s", progressbar[pgId], message );
476 uint i;
477 for (i=(uint)strlen(msg); i<79; i++)
478 msg[i]=' ';
479 msg[i]=0;
480 printf ("%s", msg);
481 printf ("\r");
482 }
483
484 int main(int argc, char* argv[])
485 {
486 if (argc==4)
487 {
488 // Load the config file
489 printf ("Loading config files...");
490
491 // Extensions
492 vector<string> extensions;
493
494 // Extensions
495 map<string, string> alternateExtensions;
496 alternateExtensions.insert (map<string, string>::value_type (".tga", ".dds"));
497
498 // String array
499 set<string> inputFiles;
500
501 // String array
502 set<string> usedFiles;
503
504 // String array
505 map<string, string> availableFiles;
506
507 if (loadConfigFiles (argv[1], argv[2], argv[3], extensions, inputFiles, availableFiles))
508 {
509 // Load the config file
510 printf ("\rScaning files... \n");
511
512 uint size = 0;
513 uint maxSize = (uint)inputFiles.size();
514 set<string>::iterator ite = inputFiles.begin();
515 while (ite != inputFiles.end())
516 {
517 // Update
518 string label = ite->c_str();
519
520 // Remove directory
521 removeDirectory (label);
522
523 progress (label.c_str(), (float)size/(float)maxSize);
524
525 // String array
526 set<string> outputFilename;
527
528 // File array
529 vector<char> fileArray;
530
531 // Read the file
532 INFO ("Open file: %s", ite->c_str());
533 if (readTheFile (ite->c_str(), fileArray))
534 {
535 // Is it a ASCII file ?
536 if (isASCII (fileArray))
537 {
538 INFO ("ASCII mode");
539
540 // Extract strings
541 extractStringsFromASCII (fileArray, outputFilename, extensions);
542 }
543 else
544 {
545 INFO ("BINARY mode");
546
547 // Extract strings
548 extractStringsFromBinary (fileArray, outputFilename, extensions);
549 }
550 }
551
552 // Check this files exists
553 set<string>::iterator found = outputFilename.begin();
554 while (found != outputFilename.end())
555 {
556 // Look for this file
557 if (availableFiles.find (*found) == availableFiles.end())
558 {
559 // Found ?
560 bool foundIt = false;
561
562 // Try alternate extension
563 string ext = getExtension ((*found).c_str());
564 std::map<string, string>::iterator alternIte = alternateExtensions.find (ext);
565 if (alternIte != alternateExtensions.end ())
566 {
567 string name = *found;
568 removeExtension (name);
569 name += alternIte->second;
570 if (availableFiles.find (name) != availableFiles.end())
571 foundIt = true;
572 }
573
574 // Not found ?
575 if (!foundIt)
576 {
577 fprintf (stderr, "MISSING: %s (needed by file %s)\n", found->c_str(), ite->c_str());
578 }
579 }
580 else
581 {
582 // Add to used files
583 usedFiles.insert (*found);
584 }
585
586 // Next file found
587 found++;
588 }
589
590 size++;
591
592 // Next file
593 ite++;
594 }
595
596 // Look for unused files
597 map<string, string>::iterator available = availableFiles.begin();
598 while (available != availableFiles.end())
599 {
600 // It is used ?
601 if (usedFiles.find (available->first) == usedFiles.end())
602 {
603 string temp = toLowerAscii (available->second);
604 fprintf (stderr, "UNUSED: %s\n", temp.c_str());
605 }
606
607 // Next
608 available++;
609 }
610
611 // Ok
612 return 1;
613 }
614 }
615 else
616 {
617 printf ("extract_filename [extensions.txt] [inputFiles.txt] [availableFiles.txt]\n");
618 }
619 return 0;
620 }