l10ntools/scripts/localize.pl

   1 :
   2 eval 'exec perl -wS $0 ${1+"$@"}'
   3     if 0;
   4
   5
   6 #*************************************************************************
   7 #
   8 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   9 #
  10 # Copyright 2000, 2010 Oracle and/or its affiliates.
  11 #
  12 # OpenOffice.org - a multi-platform office productivity suite
  13 #
  14 # This file is part of OpenOffice.org.
  15 #
  16 # OpenOffice.org is free software: you can redistribute it and/or modify
  17 # it under the terms of the GNU Lesser General Public License version 3
  18 # only, as published by the Free Software Foundation.
  19 #
  20 # OpenOffice.org is distributed in the hope that it will be useful,
  21 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  22 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23 # GNU Lesser General Public License version 3 for more details
  24 # (a copy is included in the LICENSE file that accompanied this code).
  25 #
  26 # You should have received a copy of the GNU Lesser General Public License
  27 # version 3 along with OpenOffice.org.  If not, see
  28 # <http://www.openoffice.org/license.html>
  29 # for a copy of the LGPLv3 License.
  30 #
  31 #*************************************************************************
  32
  33 use strict;
  34 use Getopt::Long;
  35 use IO::Handle;
  36 use File::Find;
  37 use File::Temp;
  38 use File::Path;
  39 use File::Copy;
  40 use File::Glob qw(:glob csh_glob);
  41 use Cwd;
  42
  43 my $CVS_BINARY = "/usr/bin/cvs";
  44 # ver 1.1
  45 #
  46 #### module lookup
  47 #use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
  48
  49 #### module lookup
  50 # OOo conform
  51 my @lib_dirs;
  52 BEGIN {
  53     if ( !defined($ENV{SOLARENV}) ) {
  54         die "No environment found (environment variable SOLARENV is undefined)";
  55     }
  56     push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
  57     push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
  58 }
  59 use lib (@lib_dirs);
  60
  61 #### globals ####
  62 my $sdffile                 = '';
  63 my $no_sort                 = '';
  64 my $create_dirs             = '';
  65 my $multi_localize_files    = '';
  66 my $module_to_merge         = '';
  67 my $sort_sdf_before         = '';
  68 my $outputfile              = '';
  69 my $no_gsicheck             = '';
  70 my $mode                    = '';
  71 my $bVerbose                = "0";
  72 my $srcpath                 = '';
  73 my $languages;
  74 #my %sl_modules;     # Contains all modules where en-US and de is source language
  75 my $use_default_date = '0';
  76 my $force_ooo_module = '0';
  77 my %is_ooo_module;
  78 my %is_so_module;
  79
  80          #         (                           leftpart                                                     )            (           rightpart                    )
  81          #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
  82 my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
  83 my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
  84 # Always use this date to prevent cvs conflicts
  85 my $default_date = "2002-02-02 02:02:02";
  86 my @sdfparticles;
  87
  88 #### main ####
  89 parse_options();
  90
  91 my $binpath = '';
  92 if( defined $ENV{UPDMINOREXT} )
  93 {
  94     $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin".$ENV{UPDMINOREXT}."/" ;
  95 }
  96 else
  97 {
  98     $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin/" ;
  99 }
 100
 101 #%sl_modules = fetch_sourcelanguage_dirlist();
 102
 103
 104 if   ( $mode eq "merge"    )    {
 105     if ( ! $no_gsicheck ){
 106         merge_gsicheck();
 107     }
 108     splitfile( $sdffile );
 109     if ( ! $no_gsicheck ){
 110         unlink $sdffile;             # remove temp file!
 111     }
 112 }
 113 elsif( $mode eq "extract"  )    {
 114     collectfiles( $outputfile );
 115 }
 116 else                            {
 117     usage();
 118 }
 119
 120 exit(0);
 121
 122 #########################################################
 123 sub splitfile{
 124
 125     my $lastFile        = '';
 126     my $currentFile     = '';
 127     my $cur_sdffile     = '';
 128     my $last_sdffile    = '';
 129     my $delim;
 130     my $badDelim;
 131     my $start           = 'TRUE';
 132     my %index  = ();
 133     my %block;
 134
 135     STDOUT->autoflush( 1 );
 136
 137     #print STDOUT "Open File $sdffile\n";
 138     open MYFILE , "< $sdffile"
 139     or die "Can't open '$sdffile'\n";
 140
 141 #    my %lang_hash;
 142     my %string_hash_ooo;
 143     my %string_hash_so;
 144     my %so_modules;
 145     $so_modules{ "extras_full" } = "TRUE";
 146
 147     while( <MYFILE>){
 148          if( /$sdf_regex/ ){
 149             my $line           = defined $_ ? $_ : '';
 150             my $prj            = defined $3 ? $3 : '';
 151             my $file           = defined $4 ? $4 : '';
 152             my $type           = defined $6 ? $6 : '';
 153             my $gid            = defined $7 ? $7 : '';
 154             my $lid            = defined $8 ? $8 : '';
 155             my $lang           = defined $12 ? $12 : '';
 156             my $plattform      = defined $10 ? $10 : '';
 157             my $helpid         = defined $9 ? $9 : '';
 158             next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
 159             chomp( $line );
 160
 161             if( $force_ooo_module )
 162             {
 163                 $string_hash_ooo { $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
 164             }
 165             else
 166             {
 167                 $string_hash_so{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
 168             }
 169         }
 170     }
 171     close( MYFILE );
 172
 173     if( !defined $ENV{SOURCE_ROOT_DIR} ){
 174         print "Error, no SOURCE_ROOT_DIR in env found.\n";
 175         exit( -1 );
 176     }
 177     my $src_root = $ENV{SOURCE_ROOT_DIR};
 178     my $ooo_src_root = $ENV{SRC_ROOT};
 179     my $so_l10n_path  = $src_root."/sun/l10n_so/source";
 180     my $ooo_l10n_path = $ooo_src_root."/l10n/source";
 181
 182     #print "$so_l10n_path\n";
 183     #print "$ooo_l10n_path\n";
 184
 185     if( $force_ooo_module )
 186     {
 187         write_sdf( \%string_hash_ooo , $ooo_l10n_path );
 188     }
 189     else
 190     {
 191         write_sdf( \%string_hash_so , $so_l10n_path );
 192     }
 193 }
 194
 195 sub write_sdf
 196 {
 197     my $string_hash         = shift;
 198     my $l10n_file           = shift;
 199
 200     foreach my $lang( keys( %{ $string_hash } ) )
 201     {
 202         my @sdf_file;
 203         next , if( $lang eq "en-US" );
 204
 205         mkdir $l10n_file."/$lang";
 206         # mkdir!!!!
 207         my $current_l10n_file = $l10n_file."/$lang/localize.sdf";
 208         print "Writing '$current_l10n_file'\n";
 209         if( open DESTFILE , "< $current_l10n_file" ){
 210
 211             while(<DESTFILE>){
 212                 if( /$sdf_regex/ ){
 213                     my $line           = defined $_ ? $_ : '';
 214                     my $prj            = defined $3 ? $3 : '';
 215                     my $file           = defined $4 ? $4 : '';
 216                     my $type           = defined $6 ? $6 : '';
 217                     my $gid            = defined $7 ? $7 : '';
 218                     my $lid            = defined $8 ? $8 : '';
 219                     my $lang           = defined $12 ? $12 : '';
 220                     my $plattform      = defined $10 ? $10 : '';
 221                     my $helpid         = defined $9 ? $9 : '';
 222
 223                     chomp( $line );
 224                     if ( defined $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } )
 225                     {
 226                         # Changed String!
 227                         push @sdf_file , $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } ;
 228                         $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = undef;
 229                     }
 230                     else
 231                     {
 232                         # No new string
 233                         push @sdf_file , $line;
 234                     }
 235                 }
 236             }
 237         }
 238         close( DESTFILE );
 239         #Now just append the enw strings
 240         #FIXME!!! Implement insertion in the correct order
 241         foreach my $key ( keys ( %{ $string_hash->{ $lang } } ) )
 242         {
 243             push @sdf_file , $string_hash->{ $lang }{ $key } , if ( defined $string_hash->{ $lang }{ $key } );
 244             #print "WARNING: Not defined = ".$string_hash->{ $lang }{ $key }."\n", if( ! defined  $string_hash->{ $lang }{ $key } );
 245         }
 246
 247         # Write the new file
 248         my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
 249         if( open DESTFILE , "+> $tmpfile " ){
 250             print DESTFILE get_license_header();
 251             foreach my $string( @sdf_file ){
 252                 print DESTFILE "$string\n";
 253             }
 254             close ( DESTFILE );
 255             if( move( $current_l10n_file , $current_l10n_file.".backup" ) ){
 256                 if( copy( $tmpfile , $current_l10n_file ) ){
 257                     unlink $l10n_file.".backup";
 258                  } else { print STDERR "Can't open/create '$l10n_file', original file is renamed to $l10n_file.backup\n"; }
 259             } else { print STDERR "Can't open/create '$l10n_file'\n"; }
 260          }else{
 261             print STDERR "WARNING: Can't open/create '$l10n_file'\n";
 262          }
 263          unlink $tmpfile;
 264      }
 265 }
 266
 267 #########################################################
 268
 269 sub get_license_header{
 270     return
 271 "#\n".
 272 "#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
 273 "#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
 274 "#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
 275 "#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
 276 "#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
 277 "#\n".
 278 "#    DO NOT EDIT! This file will be overwritten by localisation process\n".
 279 "#\n".
 280 "#*************************************************************************\n".
 281 "#\n".
 282 "# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n".
 283 "# \n".
 284 "# Copyright 2000, 2010 Oracle and/or its affiliates.\n".
 285 "#\n".
 286 "# OpenOffice.org - a multi-platform office productivity suite\n".
 287 "#\n".
 288 "# This file is part of OpenOffice.org.\n".
 289 "#\n".
 290 "# OpenOffice.org is free software: you can redistribute it and/or modify\n".
 291 "# it under the terms of the GNU Lesser General Public License version 3\n".
 292 "# only, as published by the Free Software Foundation.\n".
 293 "#\n".
 294 "# OpenOffice.org is distributed in the hope that it will be useful,\n".
 295 "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n".
 296 "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n".
 297 "# GNU Lesser General Public License version 3 for more details\n".
 298 "# (a copy is included in the LICENSE file that accompanied this code).\n".
 299 "#\n".
 300 "# You should have received a copy of the GNU Lesser General Public License\n".
 301 "# version 3 along with OpenOffice.org.  If not, see\n".
 302 "# <http://www.openoffice.org/license.html>\n".
 303 "# for a copy of the LGPLv3 License.\n".
 304 "#\n".
 305 "#*************************************************************************\n";
 306 }
 307 ######## Check input sdf file and use only the correct part
 308 sub merge_gsicheck{
 309     my $command = '';
 310     my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
 311     close ( $TMPHANDLE );
 312
 313     $command = "$ENV{WRAPCMD} " if( $ENV{WRAPCMD} );
 314     $command .= "$ENV{SOLARVER}/$ENV{INPATH}/bin/gsicheck";
 315
 316     my $errfile = $sdffile.".err";
 317     $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $sdffile";
 318     #my $rc = system( $command );
 319     my $output = `$command`;
 320     my $rc = $? << 8;
 321     if ( $output ne "" ){
 322         print STDOUT "### gsicheck ###\n";
 323         print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
 324         print STDOUT "$output\n";
 325         print STDOUT "################\n";
 326
 327     }else{
 328         # Remove the 0 Byte file
 329         unlink $errfile;
 330     }
 331     $sdffile = $tmpfile;
 332 }
 333 #########################################################
 334 # find search function
 335 sub wanted
 336 {
 337     my $file = $File::Find::name;
 338     if( -f $file && $file =~ /.*localize.sdf$/ && !( $file =~ /.*\.svn.*/ ) ) {
 339         push   @sdfparticles , $file;
 340         if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
 341         else { print ".";  }
 342     }
 343 }
 344
 345 sub add_paths
 346 {
 347     my $langhash_ref            = shift;
 348     my $root_dir = $ENV{ SRC_ROOT };
 349     my $ooo_l10n_dir = "$root_dir/l10n/source";
 350     my $so_l10n_dir  = "$root_dir/l10n_so/source";
 351
 352     if( -e $ooo_l10n_dir )
 353     {
 354         foreach my $lang ( keys( %{ $langhash_ref } ) )
 355         {
 356             my $loc_file = "$ooo_l10n_dir/$lang/localize.sdf";
 357             if( -e $loc_file )
 358             {
 359                 push @sdfparticles , "$ooo_l10n_dir/$lang/localize.sdf";
 360             }
 361             else { print "WARNING: $loc_file not found ....\n"; }
 362         }
 363     }
 364     else { die "ERROR: Can not find directory $ooo_l10n_dir!!!" }
 365     if( -e $so_l10n_dir )
 366     {
 367         foreach my $lang ( keys( %{ $langhash_ref } ) )
 368         {
 369             my $loc_file = "$so_l10n_dir/$lang/localize.sdf";
 370             if( -e $loc_file )
 371             {
 372                 push @sdfparticles , "$ooo_l10n_dir/$lang/localize.sdf";
 373             }
 374             else { #print "WARNING: $loc_file not found ....\n";
 375             }
 376         }
 377
 378     }
 379 }
 380 sub collectfiles{
 381     print STDOUT "### Localize\n";
 382     my $localizehash_ref;
 383     my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
 384
 385     # Enable autoflush on STDOUT
 386     # $| = 1;
 387     STDOUT->autoflush( 1 );
 388
 389     my $working_path = getcwd();
 390     chdir $ENV{SOURCE_ROOT_DIR}, if defined $ENV{SOURCE_ROOT_DIR};
 391     add_paths( $langhash_ref );
 392
 393     my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
 394     close( $LOCALIZEPARTICLE );
 395
 396     my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
 397     close( $ALLPARTICLES_MERGED );
 398     my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
 399     close( $LOCALIZE_LOG );
 400
 401     ## Get the localize en-US extract
 402     if( $bAll || $bUseLocalize ){
 403         print "### Fetching source language strings\n";
 404         my $command = "";
 405         my $args    = "";
 406
 407         if( $ENV{WRAPCMD} ){
 408             $command = $ENV{WRAPCMD}.$binpath."localize_sl";
 409         }else{
 410             $command = $binpath."localize_sl";
 411         }
 412         print $command;
 413         # -e
 414         # if ( -x $command ){
 415         if( $command ){
 416             if( !$bVerbose  ){ $args .= " "; }
 417             $args .= " -e -f $localizeSDF -l ";
 418             my $bFlag="";
 419             if( $bAll ) {$args .= " en-US";}
 420             else{
 421               my @list;
 422               foreach my $isokey ( keys( %{ $langhash_ref } ) ){
 423                 push @list , $isokey;
 424                 if( $langhash_ref->{ $isokey } ne "" ){
 425                     push @list , $langhash_ref->{ $isokey };
 426                 }
 427               }
 428               remove_duplicates( \@list );
 429               foreach my $isokey ( @list ){
 430                 switch :{
 431                        ( $isokey=~ /^en-US$/i  )
 432                         && do{
 433                                 if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
 434                                 else {
 435                                     $args .= "en-US";  $bFlag = "TRUE";
 436                                  }
 437                               };
 438
 439                     } #switch
 440                 } #foreach
 441               } # if
 442         } # if
 443         if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
 444
 445         my $rc = system( $command.$args );
 446
 447         if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
 448         ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
 449
 450     }
 451     ## Get sdf particles
 452 #*****************
 453     open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
 454     or die "Can't open $particleSDF_merged";
 455
 456     ## Fill fackback hash
 457     my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
 458     my %block;
 459     my $cur_fallback;
 460     if( !$bAll) {
 461         foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
 462             #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
 463             $cur_fallback = $langhash_ref->{ $cur_lang };
 464             if( $cur_fallback ne "" ){
 465                 # Insert fallback strings
 466                 #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
 467                 rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
 468             }
 469             foreach my $currentfile ( @sdfparticles ){
 470                 if ( open MYFILE , "< $currentfile" ) {
 471                     while(<MYFILE>){
 472                         if( /$sdf_regex/ ){
 473                             my $line           = defined $_ ? $_ : '';
 474                             my $prj            = defined $3 ? $3 : '';
 475                             my $file           = defined $4 ? $4 : '';
 476                             my $type           = defined $6 ? $6 : '';
 477                             my $gid            = defined $7 ? $7 : '';
 478                             my $lid            = defined $8 ? $8 : '';
 479                             my $lang           = defined $12 ? $12 : '';
 480                             my $plattform      = defined $10 ? $10 : '';
 481                             my $helpid         = defined $9 ? $9 : '';
 482
 483                             chomp( $line );
 484
 485                             if ( $lang eq $cur_lang ){
 486                                 # Overwrite fallback strings with collected strings
 487                                 #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
 488                                      $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 489                                      #}
 490
 491                             }
 492                         }
 493                     }
 494                 }else { print STDERR "WARNING: Can't open file $currentfile"; }
 495             }
 496
 497             foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
 498                 if( #$cur_lang ne "de" &&
 499                     $cur_lang ne "en-US" ){
 500                     print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
 501                 }
 502              }
 503         }
 504     } else {
 505         foreach my $currentfile ( @sdfparticles ){
 506             if ( open MYFILE , "< $currentfile" ) {
 507                 while( <MYFILE> ){
 508                     print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
 509                 }
 510             }
 511             else { print STDERR "WARNING: Can't open file $currentfile"; }
 512         }
 513     }
 514     close ALLPARTICLES_MERGED;
 515
 516     # Hash of array
 517     my %output;
 518     my @order;
 519
 520     ## Join both
 521     if( $outputfile ){
 522         if( open DESTFILE , "+> $outputfile" ){
 523             if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
 524             if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
 525
 526             # Insert localize
 527             my $extract_date="";
 528             while ( <LOCALIZEPARTICLE> ){
 529                 if( /$sdf_regex/ ){
 530                     my $leftpart       = defined $2 ? $2 : '';
 531                     my $lang           = defined $12 ? $12 : '';
 532                     my $rightpart      = defined $13 ? $13 : '';
 533                     my $timestamp      = defined $18 ? $18 : '';
 534
 535                     my $prj            = defined $3 ? $3 : '';
 536                     my $file           = defined $4 ? $4 : '';
 537                     my $type           = defined $6 ? $6 : '';
 538                     my $gid            = defined $7 ? $7 : '';
 539                     my $lid            = defined $8 ? $8 : '';
 540                     #my $lang           = defined $12 ? $12 : '';
 541                     my $plattform      = defined $10 ? $10 : '';
 542                     my $helpid         = defined $9 ? $9 : '';
 543
 544
 545                     if( $use_default_date )
 546                     {
 547                         $extract_date = "$default_date\n" ;
 548                     }
 549                     elsif( $extract_date eq "" ) {
 550                         $extract_date = $timestamp ;
 551                         $extract_date =~ tr/\r\n//d;
 552                         $extract_date .= "\n";
 553                     }
 554
 555                     if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 556                     else {
 557                         foreach my $sLang ( keys( %{ $langhash_ref } ) ){
 558                             if( $sLang=~ /all/i )                       {
 559                                 push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 560                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 561                             }
 562                             #if( $sLang eq "de" && $lang eq "de" )       {
 563                             #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 564                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 565                                 #}
 566                             if( $sLang eq "en-US" && $lang eq "en-US" ) {
 567                                 push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 568                                 if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
 569                                 #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
 570                             }
 571
 572                         }
 573                     }
 574                 }
 575             }
 576             # Insert particles
 577             while ( <ALLPARTICLES_MERGED> ){
 578                 if( /$sdf_regex/ ){
 579                     my $leftpart       = defined $2 ? $2 : '';
 580                     my $prj            = defined $3 ? $3 : '';
 581                     my $lang           = defined $12 ? $12 : '';
 582                     my $rightpart      = defined $13 ? $13 : '';
 583                     my $timestamp      = defined $18 ? $18 : '';
 584
 585                     #my $prj            = defined $3 ? $3 : '';
 586                     my $file           = defined $4 ? $4 : '';
 587                     my $type           = defined $6 ? $6 : '';
 588                     my $gid            = defined $7 ? $7 : '';
 589                     my $lid            = defined $8 ? $8 : '';
 590                     #my $lang           = defined $12 ? $12 : '';
 591                     my $plattform      = defined $10 ? $10 : '';
 592                     my $helpid         = defined $9 ? $9 : '';
 593
 594
 595                     if( $use_default_date )
 596                     {
 597                         $extract_date = "$default_date\n" ;
 598                     }
 599                     elsif( $extract_date eq "" )
 600                     {
 601                         $extract_date = $timestamp;
 602                     }
 603
 604                     if( ! ( $prj =~ /binfilter/i ) ) {
 605                         push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 606                         #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
 607                     }
 608                  }
 609             }
 610
 611             # Write!
 612             foreach my $curkey ( @order ){
 613                 foreach my $curlist ( $output{ $curkey } ){
 614                     foreach my $line ( @{$curlist} ){
 615                         print DESTFILE $line;
 616                     }
 617                 }
 618             }
 619
 620         }else { print STDERR "Can't open $outputfile";}
 621     }
 622     close DESTFILE;
 623     close LOCALIZEPARTICLE;
 624     close ALLPARTICLES_MERGED;
 625     chdir $working_path;
 626
 627     #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
 628     unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
 629
 630     #sort_outfile( $outputfile );
 631     #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
 632     }
 633
 634 #########################################################
 635 sub remove_obsolete{
 636     my $outfile = shift;
 637     my @lines;
 638     my $enusleftpart;
 639     my @good_lines;
 640
 641     print STDOUT "### Removing obsolete strings\n";
 642
 643     # Kick out all strings without en-US reference
 644     if ( open ( SORTEDFILE , "< $outfile" ) ){
 645         while( <SORTEDFILE> ){
 646             if( /$sdf_regex/ ){
 647                 my $line           = defined $_ ? $_ : '';
 648                 my $language       = defined $12 ? $12 : '';
 649                 my $prj            = defined $3 ? $3 : '';
 650                 my $file           = defined $4 ? $4 : '';
 651                 my $type           = defined $6 ? $6 : '';
 652                 my $gid            = defined $7 ? $7 : '';
 653                 my $lid            = defined $8 ? $8 : '';
 654                 my $plattform      = defined $10 ? $10 : '';
 655                 my $helpid         = defined $9 ? $9 : '';
 656
 657                 my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
 658
 659                 if( $language eq "en-US" ){                 # source string found, 1. entry
 660                     $enusleftpart = $leftpart;
 661                     push @good_lines , $line;
 662                 }else{
 663                     if( !defined $enusleftpart or !defined $leftpart ){
 664                         print STDERR "BADLINE: $line\n";
 665                         print STDERR "\$enusleftpart = $enusleftpart\n";
 666                         print STDERR "\$leftpart = $leftpart\n";
 667                     }
 668                     if( $enusleftpart eq $leftpart ){   # matching language
 669                         push @good_lines , $line;
 670                     }
 671                     #else{
 672                     #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
 673                     #}
 674                 }
 675             }
 676         }
 677         close SORTEDFILE;
 678     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 679
 680     # Write file
 681     if ( open ( SORTEDFILE , "> $outfile" ) ){
 682         foreach my $newline ( @good_lines ) {
 683             print SORTEDFILE $newline;
 684         }
 685         close SORTEDFILE;
 686     } else { print STDERR "ERROR: Can't open file $outfile\n";}
 687
 688 }
 689 #########################################################
 690 sub sort_outfile{
 691         my $outfile = shift;
 692         print STDOUT "### Sorting ... $outfile ...";
 693         my @lines;
 694         my @sorted_lines;
 695
 696
 697         #if ( open ( SORTEDFILE , "< $outputfile" ) ){
 698         if ( open ( SORTEDFILE , "< $outfile" ) ){
 699             my $line;
 700             while ( <SORTEDFILE> ){
 701                 $line = $_;
 702                 if( $line =~ /^[^\#]/ ){
 703                     push @lines , $line;
 704                 }
 705             }
 706             close SORTEDFILE;
 707             @sorted_lines = sort {
 708                 my $xa_lang          = "";
 709                 my $xa_left_part    = "";
 710                 my $xa_right_part    = "";
 711                 my $xa_timestamp     = "";
 712                 my $xb_lang          = "";
 713                 my $xb_left_part    = "";
 714                 my $xb_right_part    = "";
 715                 my $xb_timestamp     = "";
 716                 my $xa               = "";
 717                 my $xb               = "";
 718                 my @alist;
 719                 my @blist;
 720
 721                 if( $a=~ /$sdf_regex/ ){
 722                     $xa_left_part       = defined $2 ? $2 : '';
 723                     $xa_lang           = defined $12 ? $12 : '';
 724                     $xa_right_part     = defined $13 ? $13 : '';
 725                     $xa_left_part = remove_last_column( $xa_left_part );
 726
 727                 }
 728                 if( $b=~ /$sdf_regex/ ){
 729                     $xb_left_part       = defined $2 ? $2 : '';
 730                     $xb_lang           = defined $12 ? $12 : '';
 731                     $xb_right_part     = defined $13 ? $13 : '';
 732                     $xb_left_part = remove_last_column( $xb_left_part );
 733
 734
 735                 }
 736                 if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
 737                      if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
 738                          return ( $xa_right_part cmp $xb_right_part );   # Right part compare
 739                     }
 740                     elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
 741                     elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
 742                     else { return $xa_lang cmp $xb_lang; }             # lang compare
 743                 }
 744                 else {
 745                     return $xa_left_part cmp $xb_left_part;        # Left part compare
 746                 }
 747             } @lines;
 748
 749             if ( open ( SORTEDFILE , "> $outfile" ) ){
 750                 print SORTEDFILE get_license_header();
 751                 foreach my $newline ( @sorted_lines ) {
 752                     print SORTEDFILE $newline;
 753                     #print STDOUT $newline;
 754                 }
 755             }
 756             close SORTEDFILE;
 757         } else { print STDERR "WARNING: Can't open file $outfile\n";}
 758     print "done\n";
 759
 760 }
 761 #########################################################
 762 sub remove_last_column{
 763     my $string                  = shift;
 764     my @alist = split ( "\t" , $string );
 765     pop @alist;
 766     return join( "\t" , @alist );
 767 }
 768
 769 #########################################################
 770 sub rename_language{
 771     my $fallbackhashhash_ref    = shift;
 772     my $cur_fallback            = shift;
 773     my $cur_lang                = shift;
 774     my $line;
 775
 776     foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
 777         $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
 778         if( $line =~ /$sdf_regex/ ){
 779             my $leftpart       = defined $2 ? $2 : '';
 780             my $lang           = defined $12 ? $12 : '';
 781             my $rightpart      = defined $13 ? $13 : '';
 782
 783             $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
 784         }
 785     }
 786 }
 787
 788 ############################################################
 789 sub remove_duplicates{
 790     my $list_ref    = shift;
 791     my %tmphash;
 792     foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
 793     @{$list_ref} = keys( %tmphash );
 794 }
 795
 796 ##############################################################
 797 sub fetch_fallback{
 798     my $sdfparticleslist_ref   = shift;
 799     my $localizeSDF            = shift;
 800     my $langhash_ref           = shift;
 801     my %fallbackhashhash;
 802     my $cur_lang;
 803     my @langlist;
 804
 805     foreach my $key ( keys ( %{ $langhash_ref } ) ){
 806         $cur_lang = $langhash_ref->{ $key };
 807         if ( $cur_lang ne "" ) {
 808             push @langlist , $cur_lang;
 809         }
 810     }
 811     remove_duplicates( \@langlist );
 812     foreach  $cur_lang ( @langlist ){
 813         if( $cur_lang eq "en-US" ){
 814             read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
 815         }
 816     }
 817
 818     # remove de / en-US
 819     my @tmplist;
 820     foreach $cur_lang( @langlist ){
 821         if(  $cur_lang ne "en-US" ){
 822            push @tmplist , $cur_lang;
 823
 824         }
 825     }
 826     @langlist = @tmplist;
 827     if ( $#langlist +1 ){
 828         read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
 829
 830     }
 831     return (\%fallbackhashhash);
 832 }
 833
 834 #########################################################
 835 sub write_file{
 836
 837     my $localizeFile = shift;
 838     my $index_ref    = shift;
 839
 840     if( open DESTFILE , "+> $localizeFile" ){
 841         foreach my $key( %{ $index_ref } ){
 842             print DESTFILE ($index_ref->{ $key }, "\n" );
 843         }
 844         close DESTFILE;
 845     }else {
 846       print STDERR "Can't open/create '$localizeFile'";
 847     }
 848 }
 849
 850 #########################################################
 851 sub read_file{
 852
 853     my $sdffile         = shift;
 854     my $langhash_ref    = shift;
 855     my %block           = ();
 856
 857     open MYFILE , "< $sdffile"
 858         or die "Can't open '$sdffile'\n";
 859         while( <MYFILE>){
 860           if( /$sdf_regex/ ){
 861             my $line           = defined $_ ? $_ : '';
 862             my $prj            = defined $3 ? $3 : '';
 863             my $file           = defined $4 ? $4 : '';
 864             my $type           = defined $6 ? $6 : '';
 865             my $gid            = defined $7 ? $7 : '';
 866             my $lid            = defined $8 ? $8 : '';
 867             my $plattform      = defined $10 ? $10 : '';
 868             my $lang           = defined $12 ? $12 : '';
 869             my $helpid         = defined $9 ? $9 : '';
 870
 871             foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
 872                 if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
 873             }
 874         }
 875     }
 876     return (\%block);
 877 }
 878
 879 #########################################################
 880 sub read_fallbacks_from_particles{
 881
 882     my $sdfparticleslist_ref    = shift;
 883     my $isolanglist_ref         = shift;
 884     my $fallbackhashhash_ref    = shift;
 885     my $block_ref;
 886     foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
 887         if ( open MYFILE , "< $currentfile" ) {
 888             while(<MYFILE>){
 889                 if( /$sdf_regex/ ){
 890                     my $line           = defined $_ ? $_ : '';
 891                     my $prj            = defined $3 ? $3 : '';
 892                     my $file           = defined $4 ? $4 : '';
 893                     my $type           = defined $6 ? $6 : '';
 894                     my $gid            = defined $7 ? $7 : '';
 895                     my $lid            = defined $8 ? $8 : '';
 896                     my $lang           = defined $12 ? $12 : '';
 897                     my $plattform      = defined $10 ? $10 : '';
 898                     my $helpid         = defined $9 ? $9 : '';
 899
 900                     chomp( $line );
 901
 902                     foreach my $isolang ( @{$isolanglist_ref}  ){
 903                         if( $isolang=~ /$lang/i ) {
 904                             $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 905                         }
 906                     }
 907                 }
 908             }
 909        }else { print STDERR "WARNING: Can't open file $currentfile"; }
 910     }
 911 }
 912
 913 #########################################################
 914 sub read_fallbacks_from_source{
 915
 916     my $sdffile                 = shift;
 917     my $isolang                 = shift;
 918     my $fallbackhashhash_ref    = shift;
 919     my $block_ref;
 920     # read fallback for single file
 921     open MYFILE , "< $sdffile"
 922         or die "Can't open '$sdffile'\n";
 923
 924     while( <MYFILE>){
 925           if( /$sdf_regex/ ){
 926             my $line           = defined $_ ? $_ : '';
 927             my $prj            = defined $3 ? $3 : '';
 928             my $file           = defined $4 ? $4 : '';
 929             my $type           = defined $6 ? $6 : '';
 930             my $gid            = defined $7 ? $7 : '';
 931             my $lid            = defined $8 ? $8 : '';
 932             my $helpid         = defined $9 ? $9 : '';
 933             my $lang           = defined $12 ? $12 : '';
 934             my $plattform      = defined $10 ? $10 : '';
 935
 936             chomp( $line );
 937             if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
 938             }
 939         }
 940     }
 941 }
 942
 943 #########################################################
 944 sub parseLanguages{
 945
 946     my $bAll;
 947     my $bUseLocalize;
 948     my $bHasSourceLanguage="";
 949     my $bFakeEnglish="";
 950     my %langhash;
 951     my $iso="";
 952     my $fallback="";
 953
 954     #### -l all
 955     if(   $languages=~ /all/ ){
 956         $bAll = "TRUE";
 957         $bHasSourceLanguage = "TRUE";
 958     }
 959     ### -l fr=de,de
 960     elsif( $languages=~ /.*,.*/ ){
 961         my @tmpstr =  split "," , $languages;
 962         for my $lang ( @tmpstr ){
 963             if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
 964                 $iso        = $1;
 965                 $fallback   = $4;
 966
 967                 if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
 968                     $bUseLocalize = "TRUE";
 969                 }
 970                 if( ( $iso && $iso=~ /(en-US)/i ) ) {
 971                     $bHasSourceLanguage = "TRUE";
 972                 }
 973              if( $fallback ) { $langhash{ $iso } = $fallback;   }
 974              else            { $langhash{ $iso } = "";          }
 975             }
 976         }
 977     }
 978     ### -l de
 979     else{
 980         if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
 981             $iso        = $1;
 982             $fallback   = $4;
 983
 984             if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
 985                 $bUseLocalize = "TRUE";
 986
 987             }
 988             if( ( $iso && $iso=~ /(en-US)/i )  ) {
 989                 $bHasSourceLanguage = "TRUE";
 990             }
 991
 992              if( $fallback ) { $langhash{ $iso } = $fallback;   }
 993              else            { $langhash{ $iso } = "";          }
 994         }
 995     }
 996     # HACK en-US always needed!
 997     if( !$bHasSourceLanguage ){
 998         #$bHasSourceLanguage = "TRUE";
 999         $bUseLocalize = "TRUE";
1000         $bFakeEnglish = "TRUE";
1001         $langhash{ "en-US" } = "";
1002     }
1003     return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
1004 }
1005
1006 #########################################################
1007 sub parse_options{
1008
1009     my $help;
1010     my $merge;
1011     my $extract;
1012     my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
1013                              'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date , 'c' => \$create_dirs ,
1014                              'n' => \$no_gsicheck , 'o' => \$force_ooo_module );
1015     $outputfile = $sdffile;
1016
1017     #print STDOUT "DBG: lang = $languages\n";
1018     if( !$srcpath ){
1019         $srcpath = "$ENV{SRC_ROOT}";
1020         if( !$srcpath ){
1021             print STDERR "No path to the source root found!\n\n";
1022             usage();
1023             exit(1);
1024         }
1025     }
1026     if( $help ){
1027         usage();
1028         exit(0);
1029     }
1030     if( !$success || $#ARGV > 1 || ( !$sdffile ) ){
1031         usage();
1032         exit(1);
1033     }
1034     if( $merge && $sdffile && ! ( -r $sdffile)){
1035         print STDERR "Can't open file '$sdffile'\n";
1036         exit(1);
1037     }
1038     if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1039         print STDERR "Please check the -l iso code\n";
1040         exit(1);
1041     }
1042     if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1043     if( $extract ){ $mode = "extract"; }
1044     else          { $mode = "merge";   }
1045 }
1046 #my $multi_localize_files    = ''; h
1047 #my $module_to_merge         = ''; i
1048 #my $sort_sdf_before         = ''; g
1049
1050 #########################################################
1051 sub usage{
1052
1053     print STDERR "Usage: localize.pl\n";
1054     print STDERR "Split or collect SDF files\n";
1055     print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ] [ -c ]\n";
1056     print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1057     print STDERR "Options:\n";
1058     print STDERR "    -h              help\n";
1059     print STDERR "    -m              Merge mode\n";
1060     print STDERR "    -e              Extract mode\n";
1061     print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1062     print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1063     print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1064     print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1065     print STDERR "    -d              Use default date in extracted sdf file\n";
1066     print STDERR "    -c              Create needed directories\n";
1067     print STDERR "    -g              Sort sdf file before mergeing\n";
1068     print STDERR "    -h              File with localize.sdf's\n!";
1069     print STDERR "    -n              No gsicheck\n";
1070     print STDERR "    -i              Module to merge\n";
1071     print STDERR "    -o              force using ooo localization from the l10n module instead of l10n_so; \n";
1072     print STDERR "                    useful if the type can't be detected by the .svn tags; \n";
1073     print STDERR "    -v              Verbose\n";
1074     print STDERR "\nExample:\n";
1075     print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1076     print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1077 }
1078