4 #//===----------------------------------------------------------------------===//
6 #// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
7 #// See https://llvm.org/LICENSE.txt for license information.
8 #// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 #//===----------------------------------------------------------------------===//
16 use File
::Glob
":glob";
17 use Encode
qw{ encode
};
20 use lib
"$FindBin::Bin/lib";
24 our $VERSION = "0.04";
26 my $placeholder = qr{(\d)\$(s|l?[du])};
31 meta
=> { short
=> "prp" }, # "prp" stands for "property".
32 strings
=> { short
=> "str" },
33 formats
=> { short
=> "fmt" },
34 messages
=> { short
=> "msg" },
35 hints
=> { short
=> "hnt" },
37 my @sections = qw{ meta strings formats messages hints
};
38 # Assign section properties: long name, set number, base number.
39 map( $sections->{ $sections[ $_ ] }->{ long
} = $sections[ $_ ], ( 0 .. @sections - 1 ) );
40 map( $sections->{ $sections[ $_ ] }->{ set
} = ( $_ + 1 ), ( 0 .. @sections - 1 ) );
41 map( $sections->{ $sections[ $_ ] }->{ base
} = ( ( $_ + 1 ) << 16 ), ( 0 .. @sections - 1 ) );
43 # Properties of Meta section.
44 my @properties = qw{ Language Country LangId Version Revision
};
47 sub _generate_comment
($$$) {
49 my ( $data, $open, $close ) = @_;
51 $open . " Do not edit this file! " . $close . "\n" .
52 $open . " The file was generated from " . get_file
( $data->{ "%meta" }->{ source
} ) .
53 " by " . $tool . " on " . localtime() . ". " . $close . "\n";
56 }; # sub _generate_comment
61 # Convert message string to signature. Signature is a list of placeholders in sorted order.
62 # For example, signature of "%1$s value \"%2$s\" is invalid." is "%1$s %2$s".
67 while ( $msg =~ m{\G.*?$escape$placeholder}g ) {
68 $placeholders[ $1 - 1 ] = "%$1\$$2";
70 for ( my $i = 1; $i <= @placeholders; ++ $i ) {
71 if ( not defined( $placeholders[ $i - 1 ] ) ) {
72 $placeholders[ $i - 1 ] = "%$i\$-";
75 return join( " ", @placeholders );
82 # Convert message string to a C string constant.
85 if ( $target_os eq "win" ) {
86 $msg =~ s{$escape$placeholder}{\%$1!$2!}g;
101 $msg = msg2src
( $msg ); # Get windows style placeholders.
102 $msg =~ s{\\(.)}{ exists( $special->{ $1 } ) ?
$special->{ $1 } : $1 }ge;
108 sub parse_message
($) {
113 if ( $msg !~ m{\G.*?$escape}gc ) {
116 if ( $msg !~ m{\G$placeholder}gc ) {
117 return "Bad %-sequence near \"%" . substr( $msg, pos( $msg ), 7 ) . "\"";
122 }; # sub parse_message
125 sub parse_source
($) {
129 my @bulk = read_file
( $name, -layer
=> ":utf8" );
133 my $n = 0; # Line number.
134 my $obsolete = 0; # Counter of obsolete entries.
141 my ( $n, $line, $msg ) = @_;
142 runtime_error
( "Error parsing $name line $n: " . "$msg:\n" . " $line" );
145 foreach $line ( @bulk ) {
147 # Skip empty lines and comments.
148 if ( $line =~ m{\A\s*(\n|#)} ) {
152 # Parse section header.
153 if ( $line =~ m{\A-\*-\s*([A-Z_]*)\s*-\*-\s*\n\z}i ) {
154 $section = ( lc( $1 ) );
155 if ( not grep( $section eq $_, @sections ) ) {
156 $error->( $n, $line, "Unknown section \"$section\" specified" );
158 if ( exists( $data->{ $section } ) ) {
159 $error->( $n, $line, "Multiple sections of the same type specified" );
161 %idents = (); # Clean list of known message identifiers.
164 if ( not defined( $section ) ) {
165 $error->( $n, $line, "Section heading expected" );
167 # Parse section body.
168 if ( $section eq "meta" ) {
169 if ( $line =~ m{\A([A-Z_][A-Z_0-9]*)\s+"(.*)"\s*?\n?\z}i ) {
170 # Parse meta properties (such as Language, Country, and LangId).
171 my ( $property, $value ) = ( $1, $2 );
172 if ( not grep( $_ eq $property , @properties ) ) {
173 $error->( $n, $line, "Unknown property \"$property\" specified" );
175 if ( exists( $data->{ "%meta" }->{ $property } ) ) {
176 $error->( $n, $line, "Property \"$property\" has already been specified" );
178 $data->{ "%meta" }->{ $property } = $value;
182 $error->( $n, $line, "Property line expected" );
185 if ( $line =~ m{\A([A-Z_][A-Z_0-9]*)\s+"(.*)"\s*?\n?\z}i ) {
186 my ( $ident, $message ) = ( $1, $2 );
187 if ( $ident eq "OBSOLETE" ) {
188 # If id is "OBSOLETE", add a unique suffix. It provides convenient way to mark
193 if ( exists( $idents{ $ident } ) ) {
194 $error->( $n, $line, "Identifier \"$ident\" is redefined" );
197 my $err = parse_message
( $message );
199 $error->( $n, $line, $err );
202 push( @
{ $data->{ $section } }, [ $ident, $message ] );
203 $idents{ $ident } = 1;
204 $last_idx = @
{ $data->{ $section } } - 1;
207 # Parse continuation line.
208 if ( $line =~ m{\A\s*"(.*)"\s*\z} ) {
210 if ( not defined( $last_idx ) ) {
211 $error->( $n, $line, "Unexpected continuation line" );
214 my $err = parse_message
( $message );
216 $error->( $n, $line, $err );
219 $data->{ $section }->[ $last_idx ]->[ 1 ] .= $message;
222 $error->( $n, $line, "Message definition expected" );
224 $data->{ "%meta" }->{ source
} = $name;
225 foreach my $section ( @sections ) {
226 if ( not exists( $data->{ $section } ) ) {
227 $data->{ $section } = [];
229 }; # foreach $section
231 foreach my $property ( @properties ) {
232 if ( not defined( $data->{ "%meta" }->{ $property } ) ) {
234 "Error parsing $name: " .
235 "Required \"$property\" property is not specified"
238 push( @
{ $data->{ meta
} }, [ $property, $data->{ "%meta" }->{ $property } ] );
243 }; # sub parse_source
246 sub generate_enum
($$$) {
248 my ( $data, $file, $prefix ) = @_;
252 _generate_comment
( $data, "//", "//" ) .
254 "enum ${prefix}_id {\n\n" .
255 " // A special id for absence of message.\n" .
256 " ${prefix}_null = 0,\n\n";
258 foreach my $section ( @sections ) {
259 my $props = $sections->{ $section }; # Section properties.
260 my $short = $props->{ short
}; # Short section name, frequently used.
262 " // Set #$props->{ set }, $props->{ long }.\n" .
263 " ${prefix}_${short}_first = $props->{ base },\n";
264 foreach my $item ( @
{ $data->{ $section } } ) {
265 my ( $ident, undef ) = @
$item;
266 $bulk .= " ${prefix}_${short}_${ident},\n";
268 $bulk .= " ${prefix}_${short}_last,\n\n";
270 $bulk .= " ${prefix}_xxx_lastest\n\n";
273 "}; // enum ${prefix}_id\n" .
275 "typedef enum ${prefix}_id ${prefix}_id_t;\n" .
280 "// end of file //\n";
282 write_file
( $file, \
$bulk );
284 }; # sub generate_enum
287 sub generate_signature
($$) {
289 my ( $data, $file ) = @_;
292 $bulk .= "// message catalog signature file //\n\n";
294 foreach my $section ( @sections ) {
295 my $props = $sections->{ $section }; # Section properties.
296 my $short = $props->{ short
}; # Short section name, frequently used.
297 $bulk .= "-*- " . uc( $props->{ long
} ) . "-*-\n\n";
298 foreach my $item ( @
{ $data->{ $section } } ) {
299 my ( $ident, $msg ) = @
$item;
300 $bulk .= sprintf( "%-40s %s\n", $ident, msg2sgn
( $msg ) );
305 $bulk .= "// end of file //\n";
307 write_file
( $file, \
$bulk );
309 }; # sub generate_signature
312 sub generate_default
($$$) {
314 my ( $data, $file, $prefix ) = @_;
318 _generate_comment
( $data, "//", "//" ) .
321 foreach my $section ( @sections ) {
323 "static char const *\n" .
324 "__${prefix}_default_${section}" . "[] =\n" .
327 foreach my $item ( @
{ $data->{ $section } } ) {
328 my ( undef, $msg ) = @
$item;
329 $bulk .= " \"" . msg2src
( $msg ) . "\",\n";
338 "struct kmp_i18n_section {\n" .
340 " char const ** str;\n" .
341 "}; // struct kmp_i18n_section\n" .
342 "typedef struct kmp_i18n_section kmp_i18n_section_t;\n" .
344 "static kmp_i18n_section_t\n" .
345 "__${prefix}_sections[] =\n" .
348 foreach my $section ( @sections ) {
350 " { " . @
{ $data->{ $section } } . ", __${prefix}_default_${section} },\n";
358 "struct kmp_i18n_table {\n" .
360 " kmp_i18n_section_t * sect;\n" .
361 "}; // struct kmp_i18n_table\n" .
362 "typedef struct kmp_i18n_table kmp_i18n_table_t;\n" .
364 "static kmp_i18n_table_t __kmp_i18n_default_table =\n" .
366 " " . @sections . ",\n" .
367 " __kmp_i18n_sections\n" .
370 "// end of file //\n";
372 write_file
( $file, \
$bulk );
374 }; # sub generate_default
377 sub generate_message_unix
($$) {
379 my ( $data, $file ) = @_;
383 _generate_comment
( $data, "\$", "\$" ) .
387 foreach my $section ( @sections ) {
389 "\$ " . ( "-" x
78 ) . "\n\$ $section\n\$ " . ( "-" x
78 ) . "\n\n" .
390 "\$set $sections->{ $section }->{ set }\n" .
393 foreach my $item ( @
{ $data->{ $section } } ) {
394 my ( undef, $msg ) = @
$item;
396 $bulk .= "$n \"" . msg2src
( $msg ) . "\"\n";
403 "\$ end of file \$\n";
405 write_file
( $file, \
$bulk, -layer
=> ":utf8" );
407 }; # sub generate_message_linux
410 sub generate_message_windows
($$) {
412 my ( $data, $file ) = @_;
414 my $language = $data->{ "%meta" }->{ Language
};
415 my $langid = $data->{ "%meta" }->{ LangId
};
418 _generate_comment
( $data, ";", ";" ) .
420 "LanguageNames = ($language=$langid:msg_$langid)\n" .
425 foreach my $section ( @sections ) {
426 my $props = $sections->{ $section }; # Section properties.
428 " $props->{ short }=" . $props->{ set
} ."\n";
429 }; # foreach $section
433 foreach my $section ( @sections ) {
434 my $short = $sections->{ $section }->{ short
};
436 foreach my $item ( @
{ $data->{ $section } } ) {
437 my ( undef, $msg ) = @
$item;
441 "Facility=$short\n" .
442 "Language=$language\n" .
443 msg2mc
( $msg ) . "\n.\n\n";
445 }; # foreach $section
451 $bulk = encode
( "UTF-16LE", $bulk ); # Convert text to UTF-16LE used in Windows* OS.
452 write_file
( $file, \
$bulk, -binary
=> 1 );
454 }; # sub generate_message_windows
458 # Parse command line.
469 "os=s" => \
$target_os,
470 "enum-file=s" => \
$enum_file,
471 "signature-file=s" => \
$signature_file,
472 "default-file=s" => \
$default_file,
473 "message-file=s" => \
$message_file,
474 "id|lang-id" => \
$id,
475 "prefix=s" => \
$prefix,
478 cmdline_error
( "No source file specified -- nothing to do" );
481 cmdline_error
( "Too many source files specified" );
483 $input_file = $ARGV[ 0 ];
486 my $generate_message;
487 if ( $target_os =~ m{\A(?:lin|mac)\z} ) {
488 $generate_message = \
&generate_message_unix
;
489 } elsif ( $target_os eq "win" ) {
490 $generate_message = \
&generate_message_windows
;
492 runtime_error
( "OS \"$target_os\" is not supported" );
500 my $data = parse_source
( $input_file );
501 if ( defined( $id ) ) {
502 print( $data->{ "%meta" }->{ LangId
}, "\n" );
504 if ( defined( $enum_file ) ) {
505 generate_enum
( $data, $enum_file, $prefix );
507 if ( defined( $signature_file ) ) {
508 generate_signature
( $data, $signature_file );
510 if ( defined( $default_file ) ) {
511 generate_default
( $data, $default_file, $prefix );
513 if ( defined( $message_file ) ) {
514 $generate_message->( $data, $message_file );
525 B<message-converter.pl> -- Convert message catalog source file into another text forms.
529 B<message-converter.pl> I<option>... <file>
535 =item B<--enum-file=>I<file>
537 Generate enum file named I<file>.
539 =item B<--default-file=>I<file>
541 Generate default messages file named I<file>.
545 Print language identifier of the message catalog source file.
547 =item B<--message-file=>I<file>
549 Generate message file.
551 =item B<--signature-file=>I<file>
553 Generate signature file.
555 Signatures are used for checking compatibility. For example, to check a primary
556 catalog and its translation to another language, signatures of both catalogs should be generated
557 and compared. If signatures are identical, catalogs are compatible.
559 =item B<--prefix=>I<prefix>
561 Prefix to be used for all C identifiers (type and variable names) in enum and default messages
566 Specify OS name the message formats to be converted for. If not specified explicitly, value of
567 LIBOMP_OS environment variable is used. If LIBOMP_OS is not defined, host OS is detected.
569 Depending on OS, B<message-converter.pl> converts message formats to GNU style or MS style.
571 =item Standard Options
579 Print full documentation and exit.
583 Print short help message and exit.
587 Print version string and exit.
599 A name of input file.
605 =head2 Message Catalog File Format
607 It is plain text file in UTF-8 encoding. Empty lines and lines beginning with sharp sign (C<#>) are
608 ignored. EBNF syntax of content:
610 catalog = { section };
611 section = header body;
612 header = "-*- " section-id " -*-" "\n";
614 message = message-id string "\n" { string "\n" };
615 section-id = identifier;
616 message-id = "OBSOLETE" | identifier;
617 identifier = letter { letter | digit | "_" };
618 string = """ { character } """;
620 Identifier starts with letter, with following letters, digits, and underscores. Identifiers are
621 case-sensitive. Setion identifiers are fixed: C<META>, C<STRINGS>, C<FORMATS>, C<MESSAGES> and
622 C<HINTS>. Message identifiers must be unique within section. Special C<OBSOLETE> pseudo-identifier
623 may be used many times.
625 String is a C string literal which must not cross line boundaries.
626 Long messages may occupy multiple lines, a string per line.
628 Message may include printf-like GNU-style placeholders for arguments: C<%I<n>$I<t>>,
629 where I<n> is argument number (C<1>, C<2>, ...),
630 I<t> -- argument type, C<s> (string) or C<d> (32-bit integer).
632 See also comments in F<i18n/en_US.txt>.
636 This script can generate 3 different text files from single source:
642 Enum file is a C include file, containing definitions of message identifiers, e. g.:
647 kmp_i18n_prp_first = 65536,
648 kmp_i18n_prp_Language,
649 kmp_i18n_prp_Country,
651 kmp_i18n_prp_Version,
652 kmp_i18n_prp_Revision,
656 kmp_i18n_str_first = 131072,
658 kmp_i18n_str_UnknownFile,
659 kmp_i18n_str_NotANumber,
667 }; // enum kmp_i18n_id
669 typedef enum kmp_i18n_id kmp_i18n_id_t;
671 =item Default messages file.
673 Default messages file is a C include file containing default messages to be embedded into
674 application (and used if external message catalog does not exist or could not be open):
677 __kmp_i18n_default_meta[] =
689 __kmp_i18n_default_strings[] =
702 Message file is an input for message compiler, F<gencat> on Linux* OS and OS X*, or F<mc.exe> on
705 Here is the example of Linux* OS message file:
712 5 "Based on English message catalog revision 20090806"
715 Example of Windows* OS message file:
717 LanguageNames = (Japanese=10041:msg_1041)
736 Signature is a processed source file: comments stripped, strings deleted, but placeholders kept and
746 Pragma %1$s %2$s %3$s %4$s
748 The purpose of signatures -- compare two message source files for compatibility. If signatures of
749 two message sources are the same, binary message catalogs will be compatible.
755 Generate include file containing message identifiers:
757 $ message-converter.pl --enum-file=kmp_i18n_id.inc en_US.txt
759 Generate include file containing default messages:
761 $ message-converter.pl --default-file=kmp_i18n_default.inc en_US.txt
763 Generate input file for message compiler, Linux* OS example:
765 $ message-converter.pl --message-file=ru_RU.UTF-8.msg ru_RU.txt
767 Generate input file for message compiler, Windows* OS example:
769 > message-converter.pl --message-file=ru_RU.UTF-8.mc ru_RU.txt