src/tools/gen_keywordlist.pl

   1 #----------------------------------------------------------------------
   2 #
   3 # gen_keywordlist.pl
   4 #       Perl script that transforms a list of keywords into a ScanKeywordList
   5 #       data structure that can be passed to ScanKeywordLookup().
   6 #
   7 # The input is a C header file containing a series of macro calls
   8 #       PG_KEYWORD("keyword", ...)
   9 # Lines not starting with PG_KEYWORD are ignored.  The keywords are
  10 # implicitly numbered 0..N-1 in order of appearance in the header file.
  11 # Currently, the keywords are required to appear in ASCII order.
  12 #
  13 # The output is a C header file that defines a "const ScanKeywordList"
  14 # variable named according to the -v switch ("ScanKeywords" by default).
  15 # The variable is marked "static" unless the -e switch is given.
  16 #
  17 # ScanKeywordList uses hash-based lookup, so this script also selects
  18 # a minimal perfect hash function for the keyword set, and emits a
  19 # static hash function that is referenced in the ScanKeywordList struct.
  20 # The hash function is case-insensitive unless --no-case-fold is specified.
  21 # Note that case folding works correctly only for all-ASCII keywords!
  22 #
  23 #
  24 # Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
  25 # Portions Copyright (c) 1994, Regents of the University of California
  26 #
  27 # src/tools/gen_keywordlist.pl
  28 #
  29 #----------------------------------------------------------------------
  30
  31 use strict;
  32 use warnings FATAL => 'all';
  33 use Getopt::Long;
  34
  35 use FindBin;
  36 use lib $FindBin::RealBin;
  37
  38 use PerfectHash;
  39
  40 my $output_path = '';
  41 my $extern = 0;
  42 my $case_fold = 1;
  43 my $varname = 'ScanKeywords';
  44
  45 GetOptions(
  46         'output:s' => \$output_path,
  47         'extern' => \$extern,
  48         'case-fold!' => \$case_fold,
  49         'varname:s' => \$varname) || usage();
  50
  51 my $kw_input_file = shift @ARGV || die "No input file.\n";
  52
  53 # Make sure output_path ends in a slash if needed.
  54 if ($output_path ne '' && substr($output_path, -1) ne '/')
  55 {
  56         $output_path .= '/';
  57 }
  58
  59 $kw_input_file =~ /(\w+)\.h$/
  60   || die "Input file must be named something.h.\n";
  61 my $base_filename = $1 . '_d';
  62 my $kw_def_file = $output_path . $base_filename . '.h';
  63
  64 open(my $kif, '<', $kw_input_file) || die "$kw_input_file: $!\n";
  65 open(my $kwdef, '>', $kw_def_file) || die "$kw_def_file: $!\n";
  66
  67 # Opening boilerplate for keyword definition header.
  68 printf $kwdef <<EOM, $base_filename, uc $base_filename, uc $base_filename;
  69 /*-------------------------------------------------------------------------
  70  *
  71  * %s.h
  72  *    List of keywords represented as a ScanKeywordList.
  73  *
  74  * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
  75  * Portions Copyright (c) 1994, Regents of the University of California
  76  *
  77  * NOTES
  78  *  ******************************
  79  *  *** DO NOT EDIT THIS FILE! ***
  80  *  ******************************
  81  *
  82  *  It has been GENERATED by src/tools/gen_keywordlist.pl
  83  *
  84  *-------------------------------------------------------------------------
  85  */
  86
  87 #ifndef %s_H
  88 #define %s_H
  89
  90 #include "common/kwlookup.h"
  91
  92 EOM
  93
  94 # Parse input file for keyword names.
  95 my @keywords;
  96 while (<$kif>)
  97 {
  98         if (/^PG_KEYWORD\("(\w+)"/)
  99         {
 100                 push @keywords, $1;
 101         }
 102 }
 103
 104 # When being case-insensitive, insist that the input be all-lower-case.
 105 if ($case_fold)
 106 {
 107         foreach my $kw (@keywords)
 108         {
 109                 die qq|The keyword "$kw" is not lower-case in $kw_input_file\n|
 110                   if ($kw ne lc $kw);
 111         }
 112 }
 113
 114 # Error out if the keyword names are not in ASCII order.
 115 #
 116 # While this isn't really necessary with hash-based lookup, it's still
 117 # helpful because it provides a cheap way to reject duplicate keywords.
 118 # Also, insisting on sorted order ensures that code that scans the keyword
 119 # table linearly will see the keywords in a canonical order.
 120 for my $i (0 .. $#keywords - 1)
 121 {
 122         die
 123           qq|The keyword "$keywords[$i + 1]" is out of order in $kw_input_file\n|
 124           if ($keywords[$i] cmp $keywords[ $i + 1 ]) >= 0;
 125 }
 126
 127 # Emit the string containing all the keywords.
 128
 129 printf $kwdef qq|static const char %s_kw_string[] =\n\t"|, $varname;
 130 print $kwdef join qq|\\0"\n\t"|, @keywords;
 131 print $kwdef qq|";\n\n|;
 132
 133 # Emit an array of numerical offsets which will be used to index into the
 134 # keyword string.  Also determine max keyword length.
 135
 136 printf $kwdef "static const uint16 %s_kw_offsets[] = {\n", $varname;
 137
 138 my $offset = 0;
 139 my $max_len = 0;
 140 foreach my $name (@keywords)
 141 {
 142         my $this_length = length($name);
 143
 144         print $kwdef "\t$offset,\n";
 145
 146         # Calculate the cumulative offset of the next keyword,
 147         # taking into account the null terminator.
 148         $offset += $this_length + 1;
 149
 150         # Update max keyword length.
 151         $max_len = $this_length if $max_len < $this_length;
 152 }
 153
 154 print $kwdef "};\n\n";
 155
 156 # Emit a macro defining the number of keywords.
 157 # (In some places it's useful to have access to that as a constant.)
 158
 159 printf $kwdef "#define %s_NUM_KEYWORDS %d\n\n", uc $varname, scalar @keywords;
 160
 161 # Emit the definition of the hash function.
 162
 163 my $funcname = $varname . "_hash_func";
 164
 165 my $f = PerfectHash::generate_hash_function(\@keywords, $funcname,
 166         case_fold => $case_fold);
 167
 168 printf $kwdef qq|static %s\n|, $f;
 169
 170 # Emit the struct that wraps all this lookup info into one variable.
 171
 172 printf $kwdef "static " if !$extern;
 173 printf $kwdef "const ScanKeywordList %s = {\n", $varname;
 174 printf $kwdef qq|\t%s_kw_string,\n|, $varname;
 175 printf $kwdef qq|\t%s_kw_offsets,\n|, $varname;
 176 printf $kwdef qq|\t%s,\n|, $funcname;
 177 printf $kwdef qq|\t%s_NUM_KEYWORDS,\n|, uc $varname;
 178 printf $kwdef qq|\t%d\n|, $max_len;
 179 printf $kwdef "};\n\n";
 180
 181 printf $kwdef "#endif\t\t\t\t\t\t\t/* %s_H */\n", uc $base_filename;
 182
 183
 184 sub usage
 185 {
 186         die <<EOM;
 187 Usage: gen_keywordlist.pl [--output/-o <path>] [--varname/-v <varname>] [--extern/-e] [--[no-]case-fold] input_file
 188     --output        Output directory (default '.')
 189     --varname       Name for ScanKeywordList variable (default 'ScanKeywords')
 190     --extern        Allow the ScanKeywordList variable to be globally visible
 191     --no-case-fold  Keyword matching is to be case-sensitive
 192
 193 gen_keywordlist.pl transforms a list of keywords into a ScanKeywordList.
 194 The output filename is derived from the input file by inserting _d,
 195 for example kwlist_d.h is produced from kwlist.h.
 196 EOM
 197 }