l10ntools/scripts/localize_old.pl

   1 :
   2 eval 'exec perl -wS $0 ${1+"$@"}'
   3     if 0;
   4
   5
   6 #*************************************************************************
   7 #
   8 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   9 #
  10 # Copyright 2000, 2010 Oracle and/or its affiliates.
  11 #
  12 # OpenOffice.org - a multi-platform office productivity suite
  13 #
  14 # This file is part of OpenOffice.org.
  15 #
  16 # OpenOffice.org is free software: you can redistribute it and/or modify
  17 # it under the terms of the GNU Lesser General Public License version 3
  18 # only, as published by the Free Software Foundation.
  19 #
  20 # OpenOffice.org is distributed in the hope that it will be useful,
  21 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  22 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23 # GNU Lesser General Public License version 3 for more details
  24 # (a copy is included in the LICENSE file that accompanied this code).
  25 #
  26 # You should have received a copy of the GNU Lesser General Public License
  27 # version 3 along with OpenOffice.org.  If not, see
  28 # <http://www.openoffice.org/license.html>
  29 # for a copy of the LGPLv3 License.
  30 #
  31 #*************************************************************************
  32
  33 use strict;
  34 use Getopt::Long;
  35 use IO::Handle;
  36 use File::Find;
  37 use File::Temp;
  38 use File::Copy;
  39 use File::Glob qw(:glob csh_glob);
  40 use Cwd;
  41
  42 # ver 1.1
  43 #
  44 #### module lookup
  45 #use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
  46
  47 #### module lookup
  48 # OOo conform
  49 my @lib_dirs;
  50 BEGIN {
  51     if ( !defined($ENV{SOLARENV}) ) {
  52         die "No environment found (environment variable SOLARENV is undefined)";
  53     }
  54     push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
  55     push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
  56 }
  57 use lib (@lib_dirs);
  58
  59 #### globals ####
  60 my $sdffile = '';
  61 my $no_sort = '';
  62 my $outputfile = '';
  63 my $mode = '';
  64 my $bVerbose="0";
  65 my $srcpath = '';
  66 my $languages;
  67 #my %sl_modules;     # Contains all modules where en-US and de is source language
  68 my $use_default_date = '0';
  69
  70          #         (                           leftpart                                                     )            (           rightpart                    )
  71          #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
  72 my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
  73 my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
  74 # Always use this date to prevent cvs conflicts
  75 my $default_date = "2002-02-02 02:02:02";
  76
  77 #### main ####
  78 parse_options();
  79
  80 #%sl_modules = fetch_sourcelanguage_dirlist();
  81
  82
  83 if   ( $mode eq "merge"    )    {
  84     merge_gsicheck();
  85     splitfile( $sdffile );
  86     unlink $sdffile;             # remove temp file!
  87 }
  88 elsif( $mode eq "extract"  )    {
  89     collectfiles( $outputfile );
  90 }
  91 else                            {
  92     usage();
  93 }
  94
  95 exit(0);
  96
  97 #########################################################
  98 sub splitfile{
  99
 100     my $lastFile        = '';
 101     my $currentFile     = '';
 102     my $cur_sdffile     = '';
 103     my $last_sdffile    = '';
 104     my $delim;
 105     my $badDelim;
 106     my $start           = 'TRUE';
 107     my %index  = ();
 108     my %block;
 109
 110     STDOUT->autoflush( 1 );
 111
 112     #print STDOUT "Open File $sdffile\n";
 113     open MYFILE , "< $sdffile"
 114     or die "Can't open '$sdffile'\n";
 115
 116     while( <MYFILE>){
 117          if( /$sdf_regex/ ){
 118             my $line           = defined $_ ? $_ : '';
 119             my $prj            = defined $3 ? $3 : '';
 120             my $file           = defined $4 ? $4 : '';
 121             my $type           = defined $6 ? $6 : '';
 122             my $gid            = defined $7 ? $7 : '';
 123             my $lid            = defined $8 ? $8 : '';
 124             my $lang           = defined $12 ? $12 : '';
 125             my $plattform      = defined $10 ? $10 : '';
 126             my $helpid         = defined $9 ? $9 : '';
 127
 128             next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
 129             chomp( $line );
 130             $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
 131             $currentFile  =~ s/\\/\//g;
 132
 133             $cur_sdffile = $currentFile;
 134             #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
 135                 $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
 136             #}
 137
 138             # Set default date
 139             if( $line =~ /(.*)\t[^\t\$]*$/ ){
 140                 $line = $1."\t".$default_date;
 141             }
 142
 143             if( $start ){
 144                 $start='';
 145                 $lastFile = $currentFile; # ?
 146                 $last_sdffile = $cur_sdffile;
 147             }
 148
 149             if( $lang eq "en-US" ){}
 150             elsif( $cur_sdffile eq $last_sdffile )
 151             {
 152                 $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 153             }
 154             else
 155             {
 156                 writesdf( $lastFile , \%block );
 157                 $lastFile = $currentFile; #?
 158                 $last_sdffile = $cur_sdffile;
 159                 %block = ();
 160                 #if( ! $lang eq "en-US"  ) {
 161                 $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 162                 #}
 163
 164             }
 165         } #else { print STDOUT "splitfile REGEX kaputt\n";}
 166
 167     }
 168     writesdf( $lastFile , \%block );
 169     %block = ();
 170     close( MYFILE );
 171
 172 }
 173 #########################################################
 174
 175 sub writesdf{
 176
 177     my $lastFile        = shift;
 178     my $blockhash_ref   = shift;
 179     my $localizeFile    = $lastFile;
 180     my %index=();
 181
 182     if( $localizeFile =~ /\.$file_types[\s]*$/ ){
 183         $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
 184         }else {
 185             print STDERR "Strange filetype found '$localizeFile'\n";
 186             return;
 187         }
 188         if( open DESTFILE , "< $localizeFile" ){
 189
 190         #or die "Can't open/create '\$localizeFile'";
 191
 192         #### Build hash
 193         while(<DESTFILE>){
 194          if( /$sdf_regex/ ){
 195             my $line           = defined $_ ? $_ : '';
 196             my $prj            = defined $3 ? $3 : '';
 197             my $file           = defined $4 ? $4 : '';
 198             my $type           = defined $6 ? $6 : '';
 199             my $gid            = defined $7 ? $7 : '';
 200             my $lid            = defined $8 ? $8 : '';
 201             my $lang           = defined $12 ? $12 : '';
 202             my $plattform      = defined $10 ? $10 : '';
 203             my $helpid         = defined $9 ? $9 : '';
 204
 205             chomp( $line );
 206             $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
 207
 208          } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
 209
 210         }
 211         close( DESTFILE );
 212     }
 213     #### Copy new strings
 214     my @mykeys = keys( %{ $blockhash_ref } );
 215     my $isDirty = "FALSE";
 216     foreach my $key( @mykeys ){
 217         if( ! defined $index{ $key } ){
 218             # Add new entry
 219             $index{ $key }  = $blockhash_ref->{ $key} ;
 220             $isDirty = "TRUE";
 221         }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
 222             # Overwrite old entry
 223             $index{ $key } = $blockhash_ref->{ $key };
 224             $isDirty = "TRUE";
 225         }else {
 226         }
 227     }
 228
 229     #### Write file
 230
 231     if( !$bVerbose ){ print STDOUT "."; }
 232     if( $isDirty eq "TRUE" ){
 233         if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
 234         if( open DESTFILE , "+> $localizeFile" ){
 235             print DESTFILE get_license_header();
 236             @mykeys = sort keys( %index );
 237             foreach my $key( @mykeys ){
 238                 print DESTFILE ( $index{ $key } , "\n" );
 239             }
 240             close DESTFILE;
 241          }else {
 242             print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
 243             my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
 244             if( open DESTFILE , "+> $tmpfile " ){
 245                 @mykeys = keys( %index );
 246                 foreach my $key( @mykeys ){
 247                     print DESTFILE ( $index{ $key } , "\n" );
 248                 }
 249                 close DESTFILE;
 250                 if( move( $localizeFile , $localizeFile.".backup" ) ){
 251                     if( copy( $tmpfile , $localizeFile ) ){
 252                         unlink $localizeFile.".backup";
 253                     } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
 254                 } else { print STDERR "Can't open/create '$localizeFile'\n"; }
 255             }else{
 256                 print STDERR "WARNING: Can't open/create '$localizeFile'\n";
 257             }
 258             unlink $tmpfile;
 259         }
 260     }
 261 #    if( $no_sort eq '' ){
 262 #        sort_outfile( $localizeFile );
 263 #    }
 264 }
 265
 266 sub get_license_header{
 267     return
 268 "#\n".
 269 "#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
 270 "#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
 271 "#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
 272 "#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
 273 "#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
 274 "#\n".
 275 "#    DO NOT EDIT! This file will be overwritten by localisation process\n".
 276 "#\n".
 277 "#*************************************************************************\n".
 278 "#\n".
 279 "# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n".
 280 "# \n".
 281 "# Copyright 2000, 2010 Oracle and/or its affiliates.\n".
 282 "#\n".
 283 "# OpenOffice.org - a multi-platform office productivity suite\n".
 284 "#\n".
 285 "# This file is part of OpenOffice.org.\n".
 286 "#\n".
 287 "# OpenOffice.org is free software: you can redistribute it and/or modify\n".
 288 "# it under the terms of the GNU Lesser General Public License version 3\n".
 289 "# only, as published by the Free Software Foundation.\n".
 290 "#\n".
 291 "# OpenOffice.org is distributed in the hope that it will be useful,\n".
 292 "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n".
 293 "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n".
 294 "# GNU Lesser General Public License version 3 for more details\n".
 295 "# (a copy is included in the LICENSE file that accompanied this code).\n".
 296 "#\n".
 297 "# You should have received a copy of the GNU Lesser General Public License\n".
 298 "# version 3 along with OpenOffice.org.  If not, see\n".
 299 "# <http://www.openoffice.org/license.html>\n".
 300 "# for a copy of the LGPLv3 License.\n".
 301 "#\n".
 302 "#*************************************************************************\n";
 303 }
 304 ######## Check input sdf file and use only the correct part
 305 sub merge_gsicheck{
 306     my $command = '';
 307     my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
 308     my ( $TMPHANDLE2 , $tmpfile2 ) = File::Temp::tempfile();
 309     close ( $TMPHANDLE );
 310     close ( $TMPHANDLE2 );
 311
 312     unlink $tmpfile2;
 313     my $output2 = `cat $sdffile | sort > $tmpfile2`;
 314     my $rc2 = $? << 8;
 315     if( $rc2 ne  0 ){
 316         printf("ERROR: Failed -> cat $sdffile | sort > $tmpfile2\n$output2\n");
 317         exit( -1 );
 318     }
 319
 320 #    if( $ENV{WRAPCMD} ){
 321 #        $command = "$ENV{WRAPCMD} gsicheck";
 322 #    }else{
 323 #        $command = "gsicheck";
 324 #    }
 325 #    my $errfile = $tmpfile.".err";
 326 #    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $tmpfile2";
 327 #    my $output = `$command`;
 328 #    my $rc = $? << 8;
 329 #    if ( $output ne "" ){
 330 #        print STDOUT "### gsicheck ###\n";
 331 #        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
 332 #        print STDOUT "$output\n";
 333 #        print STDOUT "################\n";
 334 #
 335 #    }else{
 336 #        # Remove the 0 Byte file
 337 #        unlink $errfile;
 338 #    }
 339     $sdffile = $tmpfile2;
 340 #    unlink $tmpfile2;
 341 }
 342 #########################################################
 343 sub collectfiles{
 344     print STDOUT "### Localize\n";
 345     my @sdfparticles;
 346     my $localizehash_ref;
 347     my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
 348
 349     # Enable autoflush on STDOUT
 350     # $| = 1;
 351     STDOUT->autoflush( 1 );
 352
 353     ### Search sdf particles
 354     print STDOUT "### Searching sdf particles\n";
 355     my $working_path = getcwd();
 356     chdir $srcpath;
 357     find sub {
 358         my $file = $File::Find::name;
 359         if( -f && $file =~ /.*localize.sdf$/ ) {
 360             push   @sdfparticles , $file;
 361             if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
 362             else { print ".";  }
 363
 364          }
 365     } , getcwd() ;#"."; #$srcpath;
 366     chdir $working_path;
 367
 368     my $nFound  = $#sdfparticles +1;
 369     print "\n    $nFound files found !\n";
 370
 371     my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
 372     close( $LOCALIZEPARTICLE );
 373
 374     my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
 375     close( $ALLPARTICLES_MERGED );
 376     my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
 377     close( $LOCALIZE_LOG );
 378
 379     ## Get the localize de,en-US extract
 380     if( $bAll || $bUseLocalize ){
 381         print "### Fetching source language strings\n";
 382         my $command = "";
 383         my $args    = "";
 384
 385         if( $ENV{WRAPCMD} ){
 386             $command = "$ENV{WRAPCMD} localize_sl";
 387         }else{
 388             $command = "localize_sl";
 389         }
 390
 391         # -e
 392         # if ( -x $command ){
 393         if( $command ){
 394             if( !$bVerbose  ){ $args .= " -QQ -skip_links "; }
 395             $args .= " -e -f $localizeSDF -l ";
 396             my $bFlag="";
 397             if( $bAll ) {$args .= " en-US";}
 398             else{
 399               my @list;
 400               foreach my $isokey ( keys( %{ $langhash_ref } ) ){
 401                 push @list , $isokey;
 402                 if( $langhash_ref->{ $isokey } ne "" ){
 403                     push @list , $langhash_ref->{ $isokey };
 404                 }
 405               }
 406               remove_duplicates( \@list );
 407               foreach my $isokey ( @list ){
 408                 switch :{
 409                     #( $isokey=~ /^de$/i  )
 410                     #    && do{
 411                     #            if( $bFlag eq "TRUE" ){ $args .= ",de"; }
 412                     #            else  {
 413                     #                $args .=  "de";  $bFlag = "TRUE";
 414                     #             }
 415                     #          };
 416                         ( $isokey=~ /^en-US$/i  )
 417                         && do{
 418                                 if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
 419                                 else {
 420                                     $args .= "en-US";  $bFlag = "TRUE";
 421                                  }
 422                               };
 423
 424                     } #switch
 425                 } #foreach
 426               } # if
 427         } # if
 428         if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
 429
 430         my $rc = system( $command.$args );
 431
 432         #my $output = `$command.$args`;
 433         #my $rc = $? << 8;
 434
 435         if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
 436         ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
 437
 438     }
 439     ## Get sdf particles
 440    open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
 441     or die "Can't open $particleSDF_merged";
 442
 443     ## Fill fackback hash
 444     my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
 445 #    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF , $langhash_ref );
 446     my %block;
 447     my $cur_fallback;
 448     if( !$bAll) {
 449         foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
 450             #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
 451             $cur_fallback = $langhash_ref->{ $cur_lang };
 452             if( $cur_fallback ne "" ){
 453                 # Insert fallback strings
 454                 #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
 455                 rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
 456             }
 457             foreach my $currentfile ( @sdfparticles ){
 458                 if ( open MYFILE , "< $currentfile" ) {
 459                     while(<MYFILE>){
 460                         if( /$sdf_regex/ ){
 461                             my $line           = defined $_ ? $_ : '';
 462                             my $prj            = defined $3 ? $3 : '';
 463                             my $file           = defined $4 ? $4 : '';
 464                             my $type           = defined $6 ? $6 : '';
 465                             my $gid            = defined $7 ? $7 : '';
 466                             my $lid            = defined $8 ? $8 : '';
 467                             my $lang           = defined $12 ? $12 : '';
 468                             my $plattform      = defined $10 ? $10 : '';
 469                             my $helpid         = defined $9 ? $9 : '';
 470
 471                             chomp( $line );
 472
 473                             if ( $lang eq $cur_lang ){
 474                                 # Overwrite fallback strings with collected strings
 475                                 #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
 476                                      $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 477                                      #}
 478
 479                             }
 480                         }
 481                     }
 482                 }else { print STDERR "WARNING: Can't open file $currentfile"; }
 483             }
 484
 485             foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
 486                 if( #$cur_lang ne "de" &&
 487                     $cur_lang ne "en-US" ){
 488                     print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
 489                 }
 490              }
 491         }
 492     } else {
 493         foreach my $currentfile ( @sdfparticles ){
 494             if ( open MYFILE , "< $currentfile" ) {
 495                 while( <MYFILE> ){
 496                     print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
 497                 }
 498             }
 499             else { print STDERR "WARNING: Can't open file $currentfile"; }
 500         }
 501     }
 502     close ALLPARTICLES_MERGED;
 503
 504
 505     # Hash of array
 506     my %output;
 507     my @order;
 508
 509     ## Join both
 510     if( $outputfile ){
 511         if( open DESTFILE , "+> $outputfile" ){
 512             if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
 513             if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
 514
 515             # Insert localize
 516             my $extract_date="";
 517             while ( <LOCALIZEPARTICLE> ){
 518                 if( /$sdf_regex/ ){
 519                     my $leftpart       = defined $2 ? $2 : '';
 520                     my $lang           = defined $12 ? $12 : '';
 521                     my $rightpart      = defined $13 ? $13 : '';
 522                     my $timestamp      = defined $18 ? $18 : '';
 523
 524                     my $prj            = defined $3 ? $3 : '';
 525                     my $file           = defined $4 ? $4 : '';
 526                     my $type           = defined $6 ? $6 : '';
 527                     my $gid            = defined $7 ? $7 : '';
 528                     my $lid            = defined $8 ? $8 : '';
 529                     #my $lang           = defined $12 ? $12 : '';
 530                     my $plattform      = defined $10 ? $10 : '';
 531                     my $helpid         = defined $9 ? $9 : '';
 532
 533
 534                     if( $use_default_date )
 535                     {
 536                         $extract_date = "$default_date\n" ;
 537                     }
 538                     elsif( $extract_date eq "" ) {
 539                         $extract_date = $timestamp ;
 540                         $extract_date =~ tr/\r\n//d;
 541                         $extract_date .= "\n";
 542                     }
 543
 544                     if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 545                     else {
 546                         foreach my $sLang ( keys( %{ $langhash_ref } ) ){
 547                             if( $sLang=~ /all/i )                       {
 548                                 push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 549                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 550                             }
 551                             #if( $sLang eq "de" && $lang eq "de" )       {
 552                             #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 553                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 554                                 #}
 555                             if( $sLang eq "en-US" && $lang eq "en-US" ) {
 556                                 push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 557                                 if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 558                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 559                             }
 560
 561                         }
 562                     }
 563                 }
 564             }
 565             # Insert particles
 566             while ( <ALLPARTICLES_MERGED> ){
 567                 if( /$sdf_regex/ ){
 568                     my $leftpart       = defined $2 ? $2 : '';
 569                     my $prj            = defined $3 ? $3 : '';
 570                     my $lang           = defined $12 ? $12 : '';
 571                     my $rightpart      = defined $13 ? $13 : '';
 572                     my $timestamp      = defined $18 ? $18 : '';
 573
 574                     #my $prj            = defined $3 ? $3 : '';
 575                     my $file           = defined $4 ? $4 : '';
 576                     my $type           = defined $6 ? $6 : '';
 577                     my $gid            = defined $7 ? $7 : '';
 578                     my $lid            = defined $8 ? $8 : '';
 579                     #my $lang           = defined $12 ? $12 : '';
 580                     my $plattform      = defined $10 ? $10 : '';
 581                     my $helpid         = defined $9 ? $9 : '';
 582
 583
 584                     if( $use_default_date )
 585                     {
 586                         $extract_date = "$default_date\n" ;
 587                     }
 588                     elsif( $extract_date eq "" )
 589                     {
 590                         $extract_date = $timestamp;
 591                     }
 592
 593                     if( ! ( $prj =~ /binfilter/i ) ) {
 594                         push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 595                         #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 596                     }
 597                  }
 598             }
 599
 600             # Write!
 601             foreach my $curkey ( @order ){
 602                 foreach my $curlist ( $output{ $curkey } ){
 603                     foreach my $line ( @{$curlist} ){
 604                         print DESTFILE $line;
 605                     }
 606                 }
 607             }
 608
 609         }else { print STDERR "Can't open $outputfile";}
 610     }
 611     close DESTFILE;
 612     close LOCALIZEPARTICLE;
 613     close ALLPARTICLES_MERGED;
 614
 615     #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
 616     unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
 617
 618     #sort_outfile( $outputfile );
 619     #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
 620     }
 621
 622 #########################################################
 623 sub remove_obsolete{
 624     my $outfile = shift;
 625     my @lines;
 626     my $enusleftpart;
 627     my @good_lines;
 628
 629     print STDOUT "### Removing obsolete strings\n";
 630
 631     # Kick out all strings without en-US reference
 632     if ( open ( SORTEDFILE , "< $outfile" ) ){
 633         while( <SORTEDFILE> ){
 634             if( /$sdf_regex/ ){
 635                 my $line           = defined $_ ? $_ : '';
 636                 my $language       = defined $12 ? $12 : '';
 637                 my $prj            = defined $3 ? $3 : '';
 638                 my $file           = defined $4 ? $4 : '';
 639                 my $type           = defined $6 ? $6 : '';
 640                 my $gid            = defined $7 ? $7 : '';
 641                 my $lid            = defined $8 ? $8 : '';
 642                 my $plattform      = defined $10 ? $10 : '';
 643                 my $helpid         = defined $9 ? $9 : '';
 644
 645                 my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 646
 647                 if( $language eq "en-US" ){                 # source string found, 1. entry
 648                     $enusleftpart = $leftpart;
 649                     push @good_lines , $line;
 650                 }else{
 651                     if( !defined $enusleftpart or !defined $leftpart ){
 652                         print STDERR "BADLINE: $line\n";
 653                         print STDERR "\$enusleftpart = $enusleftpart\n";
 654                         print STDERR "\$leftpart = $leftpart\n";
 655                     }
 656                     if( $enusleftpart eq $leftpart ){   # matching language
 657                         push @good_lines , $line;
 658                     }
 659                     #else{
 660                     #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
 661                     #}
 662                 }
 663             }
 664         }
 665         close SORTEDFILE;
 666     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 667
 668     # Write file
 669     if ( open ( SORTEDFILE , "> $outfile" ) ){
 670         foreach my $newline ( @good_lines ) {
 671             print SORTEDFILE $newline;
 672         }
 673         close SORTEDFILE;
 674     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 675
 676 }
 677 #########################################################
 678 sub sort_outfile{
 679         my $outfile = shift;
 680         print STDOUT "### Sorting ... $outfile ...";
 681         my @lines;
 682         my @sorted_lines;
 683
 684
 685         #if ( open ( SORTEDFILE , "< $outputfile" ) ){
 686         if ( open ( SORTEDFILE , "< $outfile" ) ){
 687             my $line;
 688             while ( <SORTEDFILE> ){
 689                 $line = $_;
 690                 if( $line =~ /^[^\#]/ ){
 691                     push @lines , $line;
 692                 }
 693             }
 694             close SORTEDFILE;
 695             @sorted_lines = sort {
 696                 my $xa_lang          = "";
 697                 my $xa_left_part    = "";
 698                 my $xa_right_part    = "";
 699                 my $xa_timestamp     = "";
 700                 my $xb_lang          = "";
 701                 my $xb_left_part    = "";
 702                 my $xb_right_part    = "";
 703                 my $xb_timestamp     = "";
 704                 my $xa               = "";
 705                 my $xb               = "";
 706                 my @alist;
 707                 my @blist;
 708
 709                 if( $a=~ /$sdf_regex/ ){
 710                     $xa_left_part       = defined $2 ? $2 : '';
 711                     $xa_lang           = defined $12 ? $12 : '';
 712                     $xa_right_part     = defined $13 ? $13 : '';
 713                     $xa_left_part = remove_last_column( $xa_left_part );
 714
 715                 }
 716                 if( $b=~ /$sdf_regex/ ){
 717                     $xb_left_part       = defined $2 ? $2 : '';
 718                     $xb_lang           = defined $12 ? $12 : '';
 719                     $xb_right_part     = defined $13 ? $13 : '';
 720                     $xb_left_part = remove_last_column( $xb_left_part );
 721
 722
 723                 }
 724                 if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
 725                      if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
 726                          return ( $xa_right_part cmp $xb_right_part );   # Right part compare
 727                     }
 728                     elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
 729                     elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
 730                     else { return $xa_lang cmp $xb_lang; }             # lang compare
 731                 }
 732                 else {
 733                     return $xa_left_part cmp $xb_left_part;        # Left part compare
 734                 }
 735             } @lines;
 736
 737             if ( open ( SORTEDFILE , "> $outfile" ) ){
 738                 print SORTEDFILE get_license_header();
 739                 foreach my $newline ( @sorted_lines ) {
 740                     print SORTEDFILE $newline;
 741                     #print STDOUT $newline;
 742                 }
 743             }
 744             close SORTEDFILE;
 745         } else { print STDERR "WARNING: Can't open file $outfile\n";}
 746     print "done\n";
 747
 748 }
 749 #########################################################
 750 sub remove_last_column{
 751     my $string                  = shift;
 752     my @alist = split ( "\t" , $string );
 753     pop @alist;
 754     return join( "\t" , @alist );
 755 }
 756
 757 #########################################################
 758 sub rename_language{
 759     my $fallbackhashhash_ref    = shift;
 760     my $cur_fallback            = shift;
 761     my $cur_lang                = shift;
 762     my $line;
 763
 764     foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
 765         $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
 766         if( $line =~ /$sdf_regex/ ){
 767             my $leftpart       = defined $2 ? $2 : '';
 768             my $lang           = defined $12 ? $12 : '';
 769             my $rightpart      = defined $13 ? $13 : '';
 770
 771             $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
 772         }
 773     }
 774 }
 775
 776 ############################################################
 777 sub remove_duplicates{
 778     my $list_ref    = shift;
 779     my %tmphash;
 780     foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
 781     @{$list_ref} = keys( %tmphash );
 782 }
 783
 784 ##############################################################
 785 sub fetch_fallback{
 786     my $sdfparticleslist_ref   = shift;
 787     my $localizeSDF            = shift;
 788     my $langhash_ref           = shift;
 789     my %fallbackhashhash;
 790     my $cur_lang;
 791     my @langlist;
 792
 793     foreach my $key ( keys ( %{ $langhash_ref } ) ){
 794         $cur_lang = $langhash_ref->{ $key };
 795         if ( $cur_lang ne "" ) {
 796             push @langlist , $cur_lang;
 797         }
 798     }
 799     remove_duplicates( \@langlist );
 800     foreach  $cur_lang ( @langlist ){
 801         if( $cur_lang eq "en-US" ){
 802             read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
 803         }
 804     }
 805
 806     # remove de / en-US
 807     my @tmplist;
 808     foreach $cur_lang( @langlist ){
 809         if(  $cur_lang ne "en-US" ){
 810            push @tmplist , $cur_lang;
 811
 812         }
 813     }
 814     @langlist = @tmplist;
 815     if ( $#langlist +1 ){
 816         read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
 817
 818     }
 819     return (\%fallbackhashhash);
 820 }
 821
 822 #########################################################
 823 sub write_file{
 824
 825     my $localizeFile = shift;
 826     my $index_ref    = shift;
 827
 828     if( open DESTFILE , "+> $localizeFile" ){
 829         foreach my $key( %{ $index_ref } ){
 830             print DESTFILE ($index_ref->{ $key }, "\n" );
 831         }
 832         close DESTFILE;
 833     }else {
 834       print STDERR "Can't open/create '$localizeFile'";
 835     }
 836 }
 837
 838 #########################################################
 839 sub read_file{
 840
 841     my $sdffile         = shift;
 842     my $langhash_ref    = shift;
 843     my %block           = ();
 844
 845     open MYFILE , "< $sdffile"
 846         or die "Can't open '$sdffile'\n";
 847         while( <MYFILE>){
 848           if( /$sdf_regex/ ){
 849             my $line           = defined $_ ? $_ : '';
 850             my $prj            = defined $3 ? $3 : '';
 851             my $file           = defined $4 ? $4 : '';
 852             my $type           = defined $6 ? $6 : '';
 853             my $gid            = defined $7 ? $7 : '';
 854             my $lid            = defined $8 ? $8 : '';
 855             my $plattform      = defined $10 ? $10 : '';
 856             my $lang           = defined $12 ? $12 : '';
 857             my $helpid         = defined $9 ? $9 : '';
 858
 859             foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
 860                 if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
 861             }
 862         }
 863     }
 864     return (\%block);
 865 }
 866
 867 #########################################################
 868 sub read_fallbacks_from_particles{
 869
 870     my $sdfparticleslist_ref    = shift;
 871     my $isolanglist_ref         = shift;
 872     my $fallbackhashhash_ref    = shift;
 873     my $block_ref;
 874     foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
 875         if ( open MYFILE , "< $currentfile" ) {
 876             while(<MYFILE>){
 877                 if( /$sdf_regex/ ){
 878                     my $line           = defined $_ ? $_ : '';
 879                     my $prj            = defined $3 ? $3 : '';
 880                     my $file           = defined $4 ? $4 : '';
 881                     my $type           = defined $6 ? $6 : '';
 882                     my $gid            = defined $7 ? $7 : '';
 883                     my $lid            = defined $8 ? $8 : '';
 884                     my $lang           = defined $12 ? $12 : '';
 885                     my $plattform      = defined $10 ? $10 : '';
 886                     my $helpid         = defined $9 ? $9 : '';
 887
 888                     chomp( $line );
 889
 890                     foreach my $isolang ( @{$isolanglist_ref}  ){
 891                         if( $isolang=~ /$lang/i ) {
 892                             $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 893                         }
 894                     }
 895                 }
 896             }
 897        }else { print STDERR "WARNING: Can't open file $currentfile"; }
 898     }
 899 }
 900
 901 #########################################################
 902 sub read_fallbacks_from_source{
 903
 904     my $sdffile                 = shift;
 905     my $isolang                 = shift;
 906     my $fallbackhashhash_ref    = shift;
 907     my $block_ref;
 908     # read fallback for single file
 909     open MYFILE , "< $sdffile"
 910         or die "Can't open '$sdffile'\n";
 911
 912     while( <MYFILE>){
 913           if( /$sdf_regex/ ){
 914             my $line           = defined $_ ? $_ : '';
 915             my $prj            = defined $3 ? $3 : '';
 916             my $file           = defined $4 ? $4 : '';
 917             my $type           = defined $6 ? $6 : '';
 918             my $gid            = defined $7 ? $7 : '';
 919             my $lid            = defined $8 ? $8 : '';
 920             my $helpid         = defined $9 ? $9 : '';
 921             my $lang           = defined $12 ? $12 : '';
 922             my $plattform      = defined $10 ? $10 : '';
 923
 924             chomp( $line );
 925             if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 926             }
 927         }
 928     }
 929 }
 930
 931 #########################################################
 932 sub parseLanguages{
 933
 934     my $bAll;
 935     my $bUseLocalize;
 936     my $bHasSourceLanguage="";
 937     my $bFakeEnglish="";
 938     my %langhash;
 939     my $iso="";
 940     my $fallback="";
 941
 942     #### -l all
 943     if(   $languages=~ /all/ ){
 944         $bAll = "TRUE";
 945         $bHasSourceLanguage = "TRUE";
 946     }
 947     ### -l fr=de,de
 948     elsif( $languages=~ /.*,.*/ ){
 949         my @tmpstr =  split "," , $languages;
 950         for my $lang ( @tmpstr ){
 951             if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
 952                 $iso        = $1;
 953                 $fallback   = $4;
 954
 955                 if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
 956                     $bUseLocalize = "TRUE";
 957                 }
 958                 if( ( $iso && $iso=~ /(en-US)/i ) ) {
 959                     $bHasSourceLanguage = "TRUE";
 960                 }
 961              if( $fallback ) { $langhash{ $iso } = $fallback;   }
 962              else            { $langhash{ $iso } = "";          }
 963             }
 964         }
 965     }
 966     ### -l de
 967     else{
 968         if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
 969             $iso        = $1;
 970             $fallback   = $4;
 971
 972             if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
 973                 $bUseLocalize = "TRUE";
 974
 975             }
 976             if( ( $iso && $iso=~ /(en-US)/i )  ) {
 977                 $bHasSourceLanguage = "TRUE";
 978             }
 979
 980              if( $fallback ) { $langhash{ $iso } = $fallback;   }
 981              else            { $langhash{ $iso } = "";          }
 982         }
 983     }
 984     # HACK en-US always needed!
 985     if( !$bHasSourceLanguage ){
 986         #$bHasSourceLanguage = "TRUE";
 987         $bUseLocalize = "TRUE";
 988         $bFakeEnglish = "TRUE";
 989         $langhash{ "en-US" } = "";
 990     }
 991     return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
 992 }
 993
 994 #########################################################
 995 sub parse_options{
 996
 997     my $help;
 998     my $merge;
 999     my $extract;
1000     my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
1001                              'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
1002     $outputfile = $sdffile;
1003
1004     #print STDOUT "DBG: lang = $languages\n";
1005     if( !$srcpath ){
1006         #$srcpath = "$ENV{SRC_ROOT}";
1007         if( !$srcpath ){
1008             print STDERR "No path to the source root found!\n\n";
1009             usage();
1010             exit(1);
1011         }
1012     }
1013     if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1014         usage();
1015         exit(1);
1016     }
1017     if( $merge && $sdffile && ! ( -r $sdffile)){
1018         print STDERR "Can't open file '$sdffile'\n";
1019         exit(1);
1020     }
1021     if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1022         print STDERR "Please check the -l iso code\n";
1023         exit(1);
1024     }
1025     if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1026     if( $extract ){ $mode = "extract"; }
1027     else          { $mode = "merge";   }
1028 }
1029
1030 #########################################################
1031 sub usage{
1032
1033     print STDERR "Usage: localize.pl\n";
1034     print STDERR "Split or collect SDF files\n";
1035     print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1036     print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1037     print STDERR "Options:\n";
1038     print STDERR "    -h              help\n";
1039     print STDERR "    -m              Merge mode\n";
1040     print STDERR "    -e              Extract mode\n";
1041     print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1042     print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1043     print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1044     print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1045     print STDERR "    -d              Use default date in extracted sdf file\n";
1046     print STDERR "    -v              Verbose\n";
1047     print STDERR "\nExample:\n";
1048     print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1049     print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1050 }
1051
1052 #            my $line           = defined $_ ? $_ : '';
1053 #            my $leftpart       = defined $2 ? $2 : '';
1054 #            my $prj            = defined $3 ? $3 : '';
1055 #            my $file           = defined $4 ? $4 : '';
1056 #            my $dummy          = defined $5 ? $5 : '';
1057 #            my $type           = defined $6 ? $6 : '';
1058 #            my $gid            = defined $7 ? $7 : '';
1059 #            my $lid            = defined $8 ? $8 : '';
1060 #            my $helpid         = defined $9 ? $9 : '';
1061 #            my $plattform      = defined $10 ? $10 : '';
1062 #            my $width          = defined $11 ? $11 : '';
1063 #            my $lang           = defined $12 ? $12 : '';
1064 #            my $rightpart      = defined $13 ? $13 : '';
1065 #            my $text           = defined $14 ? $14 : '';
1066 #            my $helptext       = defined $15 ? $15 : '';
1067 #            my $quickhelptext  = defined $16 ? $16 : '';
1068 #            my $title          = defined $17 ? $17 : '';
1069 #            my $timestamp      = defined $18 ? $18 : '';
1070