transex3/scripts/localize.pl

   1 :
   2 eval 'exec perl -wS $0 ${1+"$@"}'
   3     if 0;
   4
   5
   6 #*************************************************************************
   7 #
   8 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   9 #
  10 # Copyright 2008 by Sun Microsystems, Inc.
  11 #
  12 # OpenOffice.org - a multi-platform office productivity suite
  13 #
  14 # $RCSfile: localize.pl,v $
  15 #
  16 # $Revision: 1.18 $
  17 #
  18 # This file is part of OpenOffice.org.
  19 #
  20 # OpenOffice.org is free software: you can redistribute it and/or modify
  21 # it under the terms of the GNU Lesser General Public License version 3
  22 # only, as published by the Free Software Foundation.
  23 #
  24 # OpenOffice.org is distributed in the hope that it will be useful,
  25 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  26 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27 # GNU Lesser General Public License version 3 for more details
  28 # (a copy is included in the LICENSE file that accompanied this code).
  29 #
  30 # You should have received a copy of the GNU Lesser General Public License
  31 # version 3 along with OpenOffice.org.  If not, see
  32 # <http://www.openoffice.org/license.html>
  33 # for a copy of the LGPLv3 License.
  34 #
  35 #*************************************************************************
  36
  37 use strict;
  38 use Getopt::Long;
  39 use IO::Handle;
  40 use File::Find;
  41 use File::Temp;
  42 use File::Copy;
  43 use File::Glob qw(:glob csh_glob);
  44 use Cwd;
  45
  46 # ver 1.1
  47 #
  48 #### module lookup
  49 #use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
  50
  51 #### module lookup
  52 # OOo conform
  53 my @lib_dirs;
  54 BEGIN {
  55     if ( !defined($ENV{SOLARENV}) ) {
  56         die "No environment found (environment variable SOLARENV is undefined)";
  57     }
  58     push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
  59     push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
  60 }
  61 use lib (@lib_dirs);
  62
  63 #### globals ####
  64 my $sdffile = '';
  65 my $no_sort = '';
  66 my $outputfile = '';
  67 my $mode = '';
  68 my $bVerbose="0";
  69 my $srcpath = '';
  70 my $WIN;
  71 my $languages;
  72 #my %sl_modules;     # Contains all modules where en-US and de is source language
  73 my $use_default_date = '0';
  74
  75          #         (                           leftpart                                                     )            (           rightpart                    )
  76          #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
  77 my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
  78 my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
  79 # Always use this date to prevent cvs conflicts
  80 my $default_date = "2002-02-02 02:02:02";
  81 my @sdfparticles;
  82
  83 #### main ####
  84 parse_options();
  85
  86 if ( defined $ENV{USE_SHELL} && $ENV{USE_SHELL} eq '4nt' ) {
  87    $WIN = 'TRUE';
  88 }
  89  else {
  90    $WIN = '';
  91 }
  92
  93 #%sl_modules = fetch_sourcelanguage_dirlist();
  94
  95
  96 if   ( $mode eq "merge"    )    {
  97     merge_gsicheck();
  98     splitfile( $sdffile );
  99     unlink $sdffile;             # remove temp file!
 100 }
 101 elsif( $mode eq "extract"  )    {
 102     collectfiles( $outputfile );
 103 }
 104 else                            {
 105     usage();
 106 }
 107
 108 exit(0);
 109
 110 #########################################################
 111 sub splitfile{
 112
 113     my $lastFile        = '';
 114     my $currentFile     = '';
 115     my $cur_sdffile     = '';
 116     my $last_sdffile    = '';
 117     my $delim;
 118     my $badDelim;
 119     my $start           = 'TRUE';
 120     my %index  = ();
 121     my %block;
 122
 123     STDOUT->autoflush( 1 );
 124
 125     #print STDOUT "Open File $sdffile\n";
 126     open MYFILE , "< $sdffile"
 127     or die "Can't open '$sdffile'\n";
 128
 129     while( <MYFILE>){
 130          if( /$sdf_regex/ ){
 131             my $line           = defined $_ ? $_ : '';
 132             my $prj            = defined $3 ? $3 : '';
 133             my $file           = defined $4 ? $4 : '';
 134             my $type           = defined $6 ? $6 : '';
 135             my $gid            = defined $7 ? $7 : '';
 136             my $lid            = defined $8 ? $8 : '';
 137             my $lang           = defined $12 ? $12 : '';
 138             my $plattform      = defined $10 ? $10 : '';
 139             my $helpid         = defined $9 ? $9 : '';
 140
 141             next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
 142                 chomp( $line );
 143             $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
 144             if ( $WIN ) { $currentFile  =~ s/\//\\/g; }
 145             else        { $currentFile  =~ s/\\/\//g; }
 146
 147             $cur_sdffile = $currentFile;
 148             #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
 149                 if( $WIN ) { $cur_sdffile =~ s/\\[^\\]*\.$file_types[\s]*$/\\localize.sdf/; }
 150                 else       { $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/; }
 151             #}
 152
 153             # Set default date
 154             if( $line =~ /(.*)\t[^\t\$]*$/ ){
 155                 $line = $1."\t".$default_date;
 156             }
 157
 158             if( $start ){
 159                 $start='';
 160                 $lastFile = $currentFile; # ?
 161                 $last_sdffile = $cur_sdffile;
 162             }
 163
 164             if( $lang eq "en-US" ){}
 165             elsif( $cur_sdffile eq $last_sdffile )
 166             {
 167                 $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 168             }
 169             else
 170             {
 171                 writesdf( $lastFile , \%block );
 172                 $lastFile = $currentFile; #?
 173                 $last_sdffile = $cur_sdffile;
 174                 %block = ();
 175                 #if( ! $lang eq "en-US"  ) {
 176                 $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 177                 #}
 178
 179             }
 180         } #else { print STDOUT "splitfile REGEX kaputt\n";}
 181
 182     }
 183     writesdf( $lastFile , \%block );
 184     %block = ();
 185     close( MYFILE );
 186
 187 }
 188 #########################################################
 189
 190 #sub fetch_sourcelanguage_dirlist
 191 #{
 192 #
 193 #    my $working_path = getcwd();
 194 #    my %sl_dirlist;
 195 #
 196 #    chdir $srcpath;
 197 #    my @all_dirs = csh_glob( "*" );
 198 #
 199 #    foreach my $file ( @all_dirs )
 200 #    {
 201 #        if( -d $file )
 202 #        {
 203 #            my $module = $file;
 204 #            $file .= "/prj/l10n";
 205 #            $file =~ s/\//\\/ , if( $WIN ) ;
 206 #
 207 #            if( -f $file )                                        # Test file <module>/prj/l10n
 208 #            {
 209 #                $sl_dirlist{ $module } = 1;
 210 #                if( $bVerbose eq "1" ) { print STDOUT "$module: de and en-US source language detected\n"; }
 211 #            }
 212 #        }
 213 #    }
 214 #
 215 #    chdir $working_path;
 216 #
 217 #    return %sl_dirlist;
 218 #}
 219
 220 #sub has_two_sourcelanguages
 221 #{
 222 #    my $module          = shift;
 223 #    return defined $sl_modules{ $module } ;
 224 #}
 225 sub writesdf{
 226
 227     my $lastFile        = shift;
 228     my $blockhash_ref   = shift;
 229     my $localizeFile    = $lastFile;
 230     my %index=();
 231
 232     if( $localizeFile =~ /\.$file_types[\s]*$/ ){
 233         if( $WIN ) { $localizeFile =~ s/\\[^\\]*\.$file_types[\s]*$/\\localize.sdf/; }
 234         else       { $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/; }
 235         }else {
 236             print STDERR "Strange filetype found '$localizeFile'\n";
 237             return;
 238         }
 239         if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
 240         if( open DESTFILE , "< $localizeFile" ){
 241
 242         #or die "Can't open/create '\$localizeFile'";
 243
 244         #### Build hash
 245         while(<DESTFILE>){
 246          if( /$sdf_regex/ ){
 247             my $line           = defined $_ ? $_ : '';
 248             my $prj            = defined $3 ? $3 : '';
 249             my $file           = defined $4 ? $4 : '';
 250             my $type           = defined $6 ? $6 : '';
 251             my $gid            = defined $7 ? $7 : '';
 252             my $lid            = defined $8 ? $8 : '';
 253             my $lang           = defined $12 ? $12 : '';
 254             my $plattform      = defined $10 ? $10 : '';
 255             my $helpid         = defined $9 ? $9 : '';
 256
 257             chomp( $line );
 258             $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 259
 260          } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
 261
 262         }
 263         close( DESTFILE );
 264     }
 265     #### Copy new strings
 266     my @mykeys = keys( %{ $blockhash_ref } );
 267     my $isDirty = "FALSE";
 268     foreach my $key( @mykeys ){
 269         if( ! defined $index{ $key } ){
 270             # Add new entry
 271             $index{ $key }  = $blockhash_ref->{ $key} ;
 272             $isDirty = "TRUE";
 273         }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
 274             # Overwrite old entry
 275             $index{ $key } = $blockhash_ref->{ $key };
 276             $isDirty = "TRUE";
 277         }else {
 278         }
 279     }
 280
 281     #### Write file
 282
 283     if( !$bVerbose ){ print STDOUT "."; }
 284     if( $isDirty eq "TRUE" ){
 285         if( open DESTFILE , "+> $localizeFile" ){
 286             print DESTFILE get_license_header();
 287             @mykeys = sort keys( %index );
 288             foreach my $key( @mykeys ){
 289                 print DESTFILE ( $index{ $key } , "\n" );
 290             }
 291             close DESTFILE;
 292          }else {
 293             print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
 294             my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
 295             if( open DESTFILE , "+> $tmpfile " ){
 296                 @mykeys = keys( %index );
 297                 foreach my $key( @mykeys ){
 298                     print DESTFILE ( $index{ $key } , "\n" );
 299                 }
 300                 close DESTFILE;
 301                 if( move( $localizeFile , $localizeFile.".backup" ) ){
 302                     if( copy( $tmpfile , $localizeFile ) ){
 303                         unlink $localizeFile.".backup";
 304                     } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
 305                 } else { print STDERR "Can't open/create '$localizeFile'\n"; }
 306             }else{
 307                 print STDERR "WARNING: Can't open/create '$localizeFile'\n";
 308             }
 309             unlink $tmpfile;
 310         }
 311     }
 312 #    if( $no_sort eq '' ){
 313 #        sort_outfile( $localizeFile );
 314 #    }
 315 }
 316
 317 sub get_license_header{
 318     return
 319 "#\n".
 320 "#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
 321 "#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
 322 "#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
 323 "#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
 324 "#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
 325 "#\n".
 326 "#    DO NOT EDIT! This file will be overwritten by localisation process\n".
 327 "#\n".
 328 "#*************************************************************************\n".
 329 "#\n".
 330 "# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n".
 331 "# \n".
 332 "# Copyright 2008 by Sun Microsystems, Inc.\n".
 333 "#\n".
 334 "# OpenOffice.org - a multi-platform office productivity suite\n".
 335 "#\n".
 336 "# \$RCSfile:".
 337 "localize.pl,v \$\n".
 338 "#\n".
 339 "# \$Revision: ".
 340 "1.17.4.1 \$\n".
 341 "#\n".
 342 "# This file is part of OpenOffice.org.\n".
 343 "#\n".
 344 "# OpenOffice.org is free software: you can redistribute it and/or modify\n".
 345 "# it under the terms of the GNU Lesser General Public License version 3\n".
 346 "# only, as published by the Free Software Foundation.\n".
 347 "#\n".
 348 "# OpenOffice.org is distributed in the hope that it will be useful,\n".
 349 "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n".
 350 "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n".
 351 "# GNU Lesser General Public License version 3 for more details\n".
 352 "# (a copy is included in the LICENSE file that accompanied this code).\n".
 353 "#\n".
 354 "# You should have received a copy of the GNU Lesser General Public License\n".
 355 "# version 3 along with OpenOffice.org.  If not, see\n".
 356 "# <http://www.openoffice.org/license.html>\n".
 357 "# for a copy of the LGPLv3 License.\n".
 358 "#\n".
 359 "#*************************************************************************\n";
 360 }
 361 ######## Check input sdf file and use only the correct part
 362 sub merge_gsicheck{
 363     my $command = '';
 364     my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
 365     close ( $TMPHANDLE );
 366
 367     if( $ENV{WRAPCMD} ){
 368         $command = "$ENV{WRAPCMD} gsicheck";
 369     }else{
 370         $command = "gsicheck";
 371     }
 372     my $errfile = $sdffile.".err";
 373     $command .= " -k -c -wcf $tmpfile -wef ".fix_cygwin_path($errfile)." -l \"\" ".fix_cygwin_path($sdffile);
 374     #my $rc = system( $command );
 375     if ($bVerbose) { print STDOUT "localize.pl running $command\n"; }
 376     my $output = `$command`;
 377     my $rc = $? << 8;
 378     if ( $output ne "" ){
 379         print STDOUT "### gsicheck ###\n";
 380         print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
 381         print STDOUT "$output\n";
 382         print STDOUT "################\n";
 383
 384     }else{
 385         # Remove the 0 Byte file
 386         unlink $errfile;
 387     }
 388     $sdffile = $tmpfile;
 389 }
 390 #########################################################
 391 # find search function
 392 sub wanted
 393 {
 394     my $file = $File::Find::name;
 395     if( -f $file && $file =~ /.*localize.sdf$/ && !( $file =~ /.*\.svn.*/ ) ) {
 396         push   @sdfparticles , $file;
 397         if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
 398         else { print ".";  }
 399     }
 400 }
 401
 402 sub collectfiles{
 403     print STDOUT "### Localize\n";
 404     my $localizehash_ref;
 405     my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
 406
 407     # Enable autoflush on STDOUT
 408     # $| = 1;
 409     STDOUT->autoflush( 1 );
 410
 411     ### Search sdf particles
 412     print STDOUT "### Searching sdf particles\n";
 413     my $working_path = getcwd();
 414     chdir $srcpath;
 415     find ( { wanted => \&wanted , follow => 1, follow_skip => 2 }, getcwd() );
 416     chdir $working_path;
 417
 418     my $nFound  = $#sdfparticles +1;
 419     print "\n    $nFound files found !\n";
 420
 421     my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
 422     close( $LOCALIZEPARTICLE );
 423
 424     my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
 425     close( $ALLPARTICLES_MERGED );
 426     my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
 427     close( $LOCALIZE_LOG );
 428
 429     ## Get the localize de,en-US extract
 430     if( $bAll || $bUseLocalize ){
 431         print "### Fetching source language strings\n";
 432         my $command = "";
 433         my $args    = "";
 434
 435         if( $ENV{WRAPCMD} ){
 436             $command = "$ENV{WRAPCMD} localize_sl";
 437         }else{
 438             $command = "localize_sl";
 439         }
 440
 441         # -e
 442         # if ( -x $command ){
 443         if( $command ){
 444             if( !$bVerbose  ){ $args .= " -QQ "; }
 445             $args .= " -e -f ".fix_cygwin_path($localizeSDF)." -l ";
 446             my $bFlag="";
 447             if( $bAll ) {$args .= " en-US";}
 448             else{
 449               my @list;
 450               foreach my $isokey ( keys( %{ $langhash_ref } ) ){
 451                 push @list , $isokey;
 452                 if( $langhash_ref->{ $isokey } ne "" ){
 453                     push @list , $langhash_ref->{ $isokey };
 454                 }
 455               }
 456               remove_duplicates( \@list );
 457               foreach my $isokey ( @list ){
 458                 switch :{
 459                        ( $isokey=~ /^en-US$/i  )
 460                         && do{
 461                                 if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
 462                                 else {
 463                                     $args .= "en-US";  $bFlag = "TRUE";
 464                                  }
 465                               };
 466
 467                     } #switch
 468                 } #foreach
 469               } # if
 470         } # if
 471 #        if ( !$bVerbose ){
 472 #            if ( $WIN eq "TRUE" ) { $args .= " > $my_localize_log";  }
 473 #            else                  { $args .= " >& $my_localize_log"; }
 474 #        }
 475         if ( $bVerbose ) { print STDOUT "localize.pl running $command.$args.\n"; }
 476
 477         my $rc = system( $command.$args );
 478
 479         if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
 480         ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
 481
 482     }
 483     ## Get sdf particles
 484    open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
 485     or die "Can't open $particleSDF_merged";
 486
 487     ## Fill fackback hash
 488     my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
 489     my %block;
 490     my $cur_fallback;
 491     if( !$bAll) {
 492         foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
 493             #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
 494             $cur_fallback = $langhash_ref->{ $cur_lang };
 495             if( $cur_fallback ne "" ){
 496                 # Insert fallback strings
 497                 #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
 498                 rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
 499             }
 500             foreach my $currentfile ( @sdfparticles ){
 501                 if ( open MYFILE , "< $currentfile" ) {
 502                     while(<MYFILE>){
 503                         if( /$sdf_regex/ ){
 504                             my $line           = defined $_ ? $_ : '';
 505                             my $prj            = defined $3 ? $3 : '';
 506                             my $file           = defined $4 ? $4 : '';
 507                             my $type           = defined $6 ? $6 : '';
 508                             my $gid            = defined $7 ? $7 : '';
 509                             my $lid            = defined $8 ? $8 : '';
 510                             my $lang           = defined $12 ? $12 : '';
 511                             my $plattform      = defined $10 ? $10 : '';
 512                             my $helpid         = defined $9 ? $9 : '';
 513
 514                             chomp( $line );
 515
 516                             if ( $lang eq $cur_lang ){
 517                                 # Overwrite fallback strings with collected strings
 518                                 #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
 519                                      $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 520                                      #}
 521
 522                             }
 523                         }
 524                     }
 525                 }else { print STDERR "WARNING: Can't open file $currentfile"; }
 526             }
 527
 528             foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
 529                 if( #$cur_lang ne "de" &&
 530                     $cur_lang ne "en-US" ){
 531                     print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
 532                 }
 533              }
 534         }
 535     } else {
 536         foreach my $currentfile ( @sdfparticles ){
 537             if ( open MYFILE , "< $currentfile" ) {
 538                 while( <MYFILE> ){
 539                     print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
 540                 }
 541             }
 542             else { print STDERR "WARNING: Can't open file $currentfile"; }
 543         }
 544     }
 545     close ALLPARTICLES_MERGED;
 546
 547
 548     # Hash of array
 549     my %output;
 550     my @order;
 551
 552     ## Join both
 553     if( $outputfile ){
 554         if( open DESTFILE , "+> $outputfile" ){
 555             if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
 556             if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
 557
 558             # Insert localize
 559             my $extract_date="";
 560             while ( <LOCALIZEPARTICLE> ){
 561                 if( /$sdf_regex/ ){
 562                     my $leftpart       = defined $2 ? $2 : '';
 563                     my $lang           = defined $12 ? $12 : '';
 564                     my $rightpart      = defined $13 ? $13 : '';
 565                     my $timestamp      = defined $18 ? $18 : '';
 566
 567                     my $prj            = defined $3 ? $3 : '';
 568                     my $file           = defined $4 ? $4 : '';
 569                     my $type           = defined $6 ? $6 : '';
 570                     my $gid            = defined $7 ? $7 : '';
 571                     my $lid            = defined $8 ? $8 : '';
 572                     #my $lang           = defined $12 ? $12 : '';
 573                     my $plattform      = defined $10 ? $10 : '';
 574                     my $helpid         = defined $9 ? $9 : '';
 575
 576
 577                     if( $use_default_date )
 578                     {
 579                         $extract_date = "$default_date\n" ;
 580                     }
 581                     elsif( $extract_date eq "" ) {
 582                         $extract_date = $timestamp ;
 583                         $extract_date =~ tr/\r\n//d;
 584                         $extract_date .= "\n";
 585                     }
 586
 587                     if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 588                     else {
 589                         foreach my $sLang ( keys( %{ $langhash_ref } ) ){
 590                             if( $sLang=~ /all/i )                       {
 591                                 push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 592                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 593                             }
 594                             #if( $sLang eq "de" && $lang eq "de" )       {
 595                             #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 596                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 597                                 #}
 598                             if( $sLang eq "en-US" && $lang eq "en-US" ) {
 599                                 push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 600                                 if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 601                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 602                             }
 603
 604                         }
 605                     }
 606                 }
 607             }
 608             # Insert particles
 609             while ( <ALLPARTICLES_MERGED> ){
 610                 if( /$sdf_regex/ ){
 611                     my $leftpart       = defined $2 ? $2 : '';
 612                     my $prj            = defined $3 ? $3 : '';
 613                     my $lang           = defined $12 ? $12 : '';
 614                     my $rightpart      = defined $13 ? $13 : '';
 615                     my $timestamp      = defined $18 ? $18 : '';
 616
 617                     #my $prj            = defined $3 ? $3 : '';
 618                     my $file           = defined $4 ? $4 : '';
 619                     my $type           = defined $6 ? $6 : '';
 620                     my $gid            = defined $7 ? $7 : '';
 621                     my $lid            = defined $8 ? $8 : '';
 622                     #my $lang           = defined $12 ? $12 : '';
 623                     my $plattform      = defined $10 ? $10 : '';
 624                     my $helpid         = defined $9 ? $9 : '';
 625
 626
 627                     if( $use_default_date )
 628                     {
 629                         $extract_date = "$default_date\n" ;
 630                     }
 631                     elsif( $extract_date eq "" )
 632                     {
 633                         $extract_date = $timestamp;
 634                     }
 635
 636                     if( ! ( $prj =~ /binfilter/i ) ) {
 637                         push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 638                         #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 639                     }
 640                  }
 641             }
 642
 643             # Write!
 644             foreach my $curkey ( @order ){
 645                 foreach my $curlist ( $output{ $curkey } ){
 646                     foreach my $line ( @{$curlist} ){
 647                         print DESTFILE $line;
 648                     }
 649                 }
 650             }
 651
 652         }else { print STDERR "Can't open $outputfile";}
 653     }
 654     close DESTFILE;
 655     close LOCALIZEPARTICLE;
 656     close ALLPARTICLES_MERGED;
 657
 658     #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
 659     unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
 660
 661     #sort_outfile( $outputfile );
 662     #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
 663     }
 664
 665 #########################################################
 666 sub remove_obsolete{
 667     my $outfile = shift;
 668     my @lines;
 669     my $enusleftpart;
 670     my @good_lines;
 671
 672     print STDOUT "### Removing obsolete strings\n";
 673
 674     # Kick out all strings without en-US reference
 675     if ( open ( SORTEDFILE , "< $outfile" ) ){
 676         while( <SORTEDFILE> ){
 677             if( /$sdf_regex/ ){
 678                 my $line           = defined $_ ? $_ : '';
 679                 my $language       = defined $12 ? $12 : '';
 680                 my $prj            = defined $3 ? $3 : '';
 681                 my $file           = defined $4 ? $4 : '';
 682                 my $type           = defined $6 ? $6 : '';
 683                 my $gid            = defined $7 ? $7 : '';
 684                 my $lid            = defined $8 ? $8 : '';
 685                 my $plattform      = defined $10 ? $10 : '';
 686                 my $helpid         = defined $9 ? $9 : '';
 687
 688                 my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 689
 690                 if( $language eq "en-US" ){                 # source string found, 1. entry
 691                     $enusleftpart = $leftpart;
 692                     push @good_lines , $line;
 693                 }else{
 694                     if( !defined $enusleftpart or !defined $leftpart ){
 695                         print STDERR "BADLINE: $line\n";
 696                         print STDERR "\$enusleftpart = $enusleftpart\n";
 697                         print STDERR "\$leftpart = $leftpart\n";
 698                     }
 699                     if( $enusleftpart eq $leftpart ){   # matching language
 700                         push @good_lines , $line;
 701                     }
 702                     #else{
 703                     #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
 704                     #}
 705                 }
 706             }
 707         }
 708         close SORTEDFILE;
 709     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 710
 711     # Write file
 712     if ( open ( SORTEDFILE , "> $outfile" ) ){
 713         foreach my $newline ( @good_lines ) {
 714             print SORTEDFILE $newline;
 715         }
 716         close SORTEDFILE;
 717     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 718
 719 }
 720 #########################################################
 721 sub sort_outfile{
 722         my $outfile = shift;
 723         print STDOUT "### Sorting ... $outfile ...";
 724         my @lines;
 725         my @sorted_lines;
 726
 727
 728         #if ( open ( SORTEDFILE , "< $outputfile" ) ){
 729         if ( open ( SORTEDFILE , "< $outfile" ) ){
 730             my $line;
 731             while ( <SORTEDFILE> ){
 732                 $line = $_;
 733                 if( $line =~ /^[^\#]/ ){
 734                     push @lines , $line;
 735                 }
 736             }
 737             close SORTEDFILE;
 738             @sorted_lines = sort {
 739                 my $xa_lang          = "";
 740                 my $xa_left_part    = "";
 741                 my $xa_right_part    = "";
 742                 my $xa_timestamp     = "";
 743                 my $xb_lang          = "";
 744                 my $xb_left_part    = "";
 745                 my $xb_right_part    = "";
 746                 my $xb_timestamp     = "";
 747                 my $xa               = "";
 748                 my $xb               = "";
 749                 my @alist;
 750                 my @blist;
 751
 752                 if( $a=~ /$sdf_regex/ ){
 753                     $xa_left_part       = defined $2 ? $2 : '';
 754                     $xa_lang           = defined $12 ? $12 : '';
 755                     $xa_right_part     = defined $13 ? $13 : '';
 756                     $xa_left_part = remove_last_column( $xa_left_part );
 757
 758                 }
 759                 if( $b=~ /$sdf_regex/ ){
 760                     $xb_left_part       = defined $2 ? $2 : '';
 761                     $xb_lang           = defined $12 ? $12 : '';
 762                     $xb_right_part     = defined $13 ? $13 : '';
 763                     $xb_left_part = remove_last_column( $xb_left_part );
 764
 765
 766                 }
 767                 if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
 768                      if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
 769                          return ( $xa_right_part cmp $xb_right_part );   # Right part compare
 770                     }
 771                     elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
 772                     elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
 773                     else { return $xa_lang cmp $xb_lang; }             # lang compare
 774                 }
 775                 else {
 776                     return $xa_left_part cmp $xb_left_part;        # Left part compare
 777                 }
 778             } @lines;
 779
 780             if ( open ( SORTEDFILE , "> $outfile" ) ){
 781                 print SORTEDFILE get_license_header();
 782                 foreach my $newline ( @sorted_lines ) {
 783                     print SORTEDFILE $newline;
 784                     #print STDOUT $newline;
 785                 }
 786             }
 787             close SORTEDFILE;
 788         } else { print STDERR "WARNING: Can't open file $outfile\n";}
 789         print "done\n";
 790
 791 }
 792 #########################################################
 793 sub remove_last_column{
 794     my $string                  = shift;
 795     my @alist = split ( "\t" , $string );
 796     pop @alist;
 797     return join( "\t" , @alist );
 798 }
 799
 800 #########################################################
 801 sub rename_language{
 802     my $fallbackhashhash_ref    = shift;
 803     my $cur_fallback            = shift;
 804     my $cur_lang                = shift;
 805     my $line;
 806
 807     foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
 808         $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
 809         if( $line =~ /$sdf_regex/ ){
 810             my $leftpart       = defined $2 ? $2 : '';
 811             my $lang           = defined $12 ? $12 : '';
 812             my $rightpart      = defined $13 ? $13 : '';
 813
 814             $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
 815         }
 816     }
 817 }
 818
 819 ############################################################
 820 sub remove_duplicates{
 821     my $list_ref    = shift;
 822     my %tmphash;
 823     foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
 824     @{$list_ref} = keys( %tmphash );
 825 }
 826
 827 ##############################################################
 828 sub fetch_fallback{
 829     my $sdfparticleslist_ref   = shift;
 830     my $localizeSDF            = shift;
 831     my $langhash_ref           = shift;
 832     my %fallbackhashhash;
 833     my $cur_lang;
 834     my @langlist;
 835
 836     foreach my $key ( keys ( %{ $langhash_ref } ) ){
 837         $cur_lang = $langhash_ref->{ $key };
 838         if ( $cur_lang ne "" ) {
 839             push @langlist , $cur_lang;
 840         }
 841     }
 842     remove_duplicates( \@langlist );
 843     foreach  $cur_lang ( @langlist ){
 844         if( $cur_lang eq "en-US" ){
 845             read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
 846         }
 847     }
 848
 849     # remove de / en-US
 850     my @tmplist;
 851     foreach $cur_lang( @langlist ){
 852         if(  $cur_lang ne "en-US" ){
 853            push @tmplist , $cur_lang;
 854
 855         }
 856     }
 857     @langlist = @tmplist;
 858     if ( $#langlist +1 ){
 859         read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
 860
 861     }
 862     return (\%fallbackhashhash);
 863 }
 864
 865 #########################################################
 866 sub write_file{
 867
 868     my $localizeFile = shift;
 869     my $index_ref    = shift;
 870
 871     if( open DESTFILE , "+> $localizeFile" ){
 872         foreach my $key( %{ $index_ref } ){
 873             print DESTFILE ($index_ref->{ $key }, "\n" );
 874         }
 875         close DESTFILE;
 876     }else {
 877       print STDERR "Can't open/create '$localizeFile'";
 878     }
 879 }
 880
 881 #########################################################
 882 sub read_file{
 883
 884     my $sdffile         = shift;
 885     my $langhash_ref    = shift;
 886     my %block           = ();
 887
 888     open MYFILE , "< $sdffile"
 889         or die "Can't open '$sdffile'\n";
 890         while( <MYFILE>){
 891           if( /$sdf_regex/ ){
 892             my $line           = defined $_ ? $_ : '';
 893             my $prj            = defined $3 ? $3 : '';
 894             my $file           = defined $4 ? $4 : '';
 895             my $type           = defined $6 ? $6 : '';
 896             my $gid            = defined $7 ? $7 : '';
 897             my $lid            = defined $8 ? $8 : '';
 898             my $plattform      = defined $10 ? $10 : '';
 899             my $lang           = defined $12 ? $12 : '';
 900             my $helpid         = defined $9 ? $9 : '';
 901
 902             foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
 903                 if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
 904             }
 905         }
 906     }
 907     return (\%block);
 908 }
 909
 910 #########################################################
 911 sub read_fallbacks_from_particles{
 912
 913     my $sdfparticleslist_ref    = shift;
 914     my $isolanglist_ref         = shift;
 915     my $fallbackhashhash_ref    = shift;
 916     my $block_ref;
 917     foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
 918         if ( open MYFILE , "< $currentfile" ) {
 919             while(<MYFILE>){
 920                 if( /$sdf_regex/ ){
 921                     my $line           = defined $_ ? $_ : '';
 922                     my $prj            = defined $3 ? $3 : '';
 923                     my $file           = defined $4 ? $4 : '';
 924                     my $type           = defined $6 ? $6 : '';
 925                     my $gid            = defined $7 ? $7 : '';
 926                     my $lid            = defined $8 ? $8 : '';
 927                     my $lang           = defined $12 ? $12 : '';
 928                     my $plattform      = defined $10 ? $10 : '';
 929                     my $helpid         = defined $9 ? $9 : '';
 930
 931                     chomp( $line );
 932
 933                     foreach my $isolang ( @{$isolanglist_ref}  ){
 934                         if( $isolang=~ /$lang/i ) {
 935                             $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 936                         }
 937                     }
 938                 }
 939             }
 940        }else { print STDERR "WARNING: Can't open file $currentfile"; }
 941     }
 942 }
 943
 944 #########################################################
 945 sub read_fallbacks_from_source{
 946
 947     my $sdffile                 = shift;
 948     my $isolang                 = shift;
 949     my $fallbackhashhash_ref    = shift;
 950     my $block_ref;
 951     # read fallback for single file
 952     open MYFILE , "< $sdffile"
 953         or die "Can't open '$sdffile'\n";
 954
 955     while( <MYFILE>){
 956           if( /$sdf_regex/ ){
 957             my $line           = defined $_ ? $_ : '';
 958             my $prj            = defined $3 ? $3 : '';
 959             my $file           = defined $4 ? $4 : '';
 960             my $type           = defined $6 ? $6 : '';
 961             my $gid            = defined $7 ? $7 : '';
 962             my $lid            = defined $8 ? $8 : '';
 963             my $helpid         = defined $9 ? $9 : '';
 964             my $lang           = defined $12 ? $12 : '';
 965             my $plattform      = defined $10 ? $10 : '';
 966
 967             chomp( $line );
 968             if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 969             }
 970         }
 971     }
 972 }
 973
 974 #########################################################
 975 sub parseLanguages{
 976
 977     my $bAll;
 978     my $bUseLocalize;
 979     my $bHasSourceLanguage="";
 980     my $bFakeEnglish="";
 981     my %langhash;
 982     my $iso="";
 983     my $fallback="";
 984
 985     #### -l all
 986     if(   $languages=~ /all/ ){
 987         $bAll = "TRUE";
 988         $bHasSourceLanguage = "TRUE";
 989     }
 990     ### -l fr=de,de
 991     elsif( $languages=~ /.*,.*/ ){
 992         my @tmpstr =  split "," , $languages;
 993         for my $lang ( @tmpstr ){
 994             if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
 995                 $iso        = $1;
 996                 $fallback   = $4;
 997
 998                 if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
 999                     $bUseLocalize = "TRUE";
1000                 }
1001                 if( ( $iso && $iso=~ /(en-US)/i ) ) {
1002                     $bHasSourceLanguage = "TRUE";
1003                 }
1004              if( $fallback ) { $langhash{ $iso } = $fallback;   }
1005              else            { $langhash{ $iso } = "";          }
1006             }
1007         }
1008     }
1009     ### -l de
1010     else{
1011         if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
1012             $iso        = $1;
1013             $fallback   = $4;
1014
1015             if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
1016                 $bUseLocalize = "TRUE";
1017
1018             }
1019             if( ( $iso && $iso=~ /(en-US)/i )  ) {
1020                 $bHasSourceLanguage = "TRUE";
1021             }
1022
1023              if( $fallback ) { $langhash{ $iso } = $fallback;   }
1024              else            { $langhash{ $iso } = "";          }
1025         }
1026     }
1027     # HACK en-US always needed!
1028     if( !$bHasSourceLanguage ){
1029         #$bHasSourceLanguage = "TRUE";
1030         $bUseLocalize = "TRUE";
1031         $bFakeEnglish = "TRUE";
1032         $langhash{ "en-US" } = "";
1033     }
1034     return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
1035 }
1036
1037 #########################################################
1038 sub parse_options{
1039
1040     my $help;
1041     my $merge;
1042     my $extract;
1043     my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
1044                              'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
1045     $outputfile = $sdffile;
1046
1047     #print STDOUT "DBG: lang = $languages\n";
1048     if( !$srcpath ){
1049         $srcpath = "$ENV{SRC_ROOT}";
1050         if( !$srcpath ){
1051                 print STDERR "No path to the source root found!\n\n";
1052                 usage();
1053             exit(1);
1054         }
1055     }
1056     if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1057         usage();
1058         exit(1);
1059     }
1060     if( $merge && $sdffile && ! ( -r $sdffile)){
1061         print STDERR "Can't open file '$sdffile'\n";
1062         exit(1);
1063     }
1064     if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1065         print STDERR "Please check the -l iso code\n";
1066         exit(1);
1067     }
1068     if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1069     if( $extract ){ $mode = "extract"; }
1070     else          { $mode = "merge";   }
1071 }
1072
1073 #########################################################
1074 sub usage{
1075
1076     print STDERR "Usage: localize.pl\n";
1077     print STDERR "Split or collect SDF files\n";
1078     print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1079     print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1080     print STDERR "Options:\n";
1081     print STDERR "    -h              help\n";
1082     print STDERR "    -m              Merge mode\n";
1083     print STDERR "    -e              Extract mode\n";
1084     print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1085     print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1086     print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1087     print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1088     print STDERR "    -d              Use default date in extracted sdf file\n";
1089     print STDERR "    -v              Verbose\n";
1090     print STDERR "\nExample:\n";
1091     print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1092     print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1093 }
1094
1095 sub fix_cygwin_path
1096 {
1097     my ( $path ) = @_;
1098
1099     if ( $^O eq 'cygwin' )
1100     {
1101         $path = qx{cygpath -m "$path"};
1102         chomp($path);
1103     }
1104
1105     return $path;
1106 }
1107
1108 #            my $line           = defined $_ ? $_ : '';
1109 #            my $leftpart       = defined $2 ? $2 : '';
1110 #            my $prj            = defined $3 ? $3 : '';
1111 #            my $file           = defined $4 ? $4 : '';
1112 #            my $dummy          = defined $5 ? $5 : '';
1113 #            my $type           = defined $6 ? $6 : '';
1114 #            my $gid            = defined $7 ? $7 : '';
1115 #            my $lid            = defined $8 ? $8 : '';
1116 #            my $helpid         = defined $9 ? $9 : '';
1117 #            my $plattform      = defined $10 ? $10 : '';
1118 #            my $width          = defined $11 ? $11 : '';
1119 #            my $lang           = defined $12 ? $12 : '';
1120 #            my $rightpart      = defined $13 ? $13 : '';
1121 #            my $text           = defined $14 ? $14 : '';
1122 #            my $helptext       = defined $15 ? $15 : '';
1123 #            my $quickhelptext  = defined $16 ? $16 : '';
1124 #            my $title          = defined $17 ? $17 : '';
1125 #            my $timestamp      = defined $18 ? $18 : '';
1126