patches/curl/q/t_mk-ca-bundle_improvements.diff

   1 Subject: [PATCH] mk-ca-bundle.pl: various improvements
   2
   3 Signed-off-by: Kyle J. McKay <mackyle@gmail.com>
   4
   5 ---
   6  lib/mk-ca-bundle.pl | 207 +++++++++++++++++++++++++++++++---------------------
   7  1 file changed, 122 insertions(+), 85 deletions(-)
   8
   9 diff --git a/lib/mk-ca-bundle.pl b/lib/mk-ca-bundle.pl
  10 index 9574f1db..764f153e 100755
  11 --- a/lib/mk-ca-bundle.pl
  12 +++ b/lib/mk-ca-bundle.pl
  13 @@ -8,6 +8,9 @@
  14  # *
  15  # * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
  16  # *
  17 +# * Additional modifications copyright (C) 2014,2015,2016 Kyle J. McKay.
  18 +# * All rights reserved.
  19 +# *
  20  # * This software is licensed as described in the file COPYING, which
  21  # * you should have received as part of this distribution. The terms
  22  # * are also available at https://curl.haxx.se/docs/copyright.html.
  23 @@ -34,16 +37,33 @@ use Encode;
  24  use Getopt::Std;
  25  use MIME::Base64;
  26  use strict;
  27 -use vars qw($opt_b $opt_d $opt_f $opt_h $opt_i $opt_k $opt_l $opt_m $opt_n $opt_p $opt_q $opt_s $opt_t $opt_u $opt_v $opt_w);
  28 +use vars qw($opt_b $opt_d $opt_f $opt_h $opt_i $opt_k $opt_l $opt_m $opt_n $opt_p $opt_q $opt_s $opt_t $opt_u $opt_v $opt_w $have_lwp $have_sha1);
  29  use List::Util;
  30  use Text::Wrap;
  31 -my $MOD_SHA = "Digest::SHA";
  32 -eval "require $MOD_SHA";
  33 -if ($@) {
  34 -  $MOD_SHA = "Digest::SHA::PurePerl";
  35 -  eval "require $MOD_SHA";
  36 +BEGIN {
  37 +  eval {
  38 +    require LWP::UserAgent;
  39 +    LWP::UserAgent->import;
  40 +    $have_lwp = 1;
  41 +  };
  42 +}
  43 +BEGIN {
  44 +  $have_sha1 = 1;
  45 +  eval {
  46 +    require Digest::SHA;
  47 +    Digest::SHA->import(qw(sha1_hex));
  48 +    1;
  49 +  } || eval {
  50 +    require Digest::SHA1;
  51 +    Digest::SHA1->import(qw(sha1_hex));
  52 +    1;
  53 +  } || eval {
  54 +    require Digest::SHA::PurePerl;
  55 +    Digest::SHA::PurePerl->import(qw(sha1_hex));
  56 +    1;
  57 +  } ||
  58 +  undef($have_sha1);
  59  }
  60 -eval "require LWP::UserAgent";
  61
  62  my %urls = (
  63    'nss' =>
  64 @@ -63,7 +83,7 @@ $opt_d = 'release';
  65  # If the OpenSSL commandline is not in search path you can configure it here!
  66  my $openssl = 'openssl';
  67
  68 -my $version = '1.27';
  69 +my $version = '1.27_1';
  70
  71  $opt_w = 76; # default base64 encoded lines length
  72
  73 @@ -122,14 +142,14 @@ my $url;
  74  if(defined($urls{$opt_d})) {
  75    $url = $urls{$opt_d};
  76    if(!$opt_k && $url !~ /^https:\/\//i) {
  77 -    die "The URL for '$opt_d' is not HTTPS. Use -k to override (insecure).\n";
  78 +    die "The URL for '$opt_d' is not https:. Use -k to override (insecure).\n";
  79    }
  80  }
  81  else {
  82    $url = $opt_d;
  83  }
  84
  85 -my $curl = `curl -V`;
  86 +my $curl = `curl -V 2>/dev/null` || '';
  87
  88  if ($opt_i) {
  89    print ("=" x 78 . "\n");
  90 @@ -138,10 +158,12 @@ if ($opt_i) {
  91    print "Operating System Name            : $^O\n";
  92    print "Getopt::Std.pm Version           : ${Getopt::Std::VERSION}\n";
  93    print "MIME::Base64.pm Version          : ${MIME::Base64::VERSION}\n";
  94 -  print "LWP::UserAgent.pm Version        : ${LWP::UserAgent::VERSION}\n" if($LWP::UserAgent::VERSION);
  95 -  print "LWP.pm Version                   : ${LWP::VERSION}\n" if($LWP::VERSION);
  96 -  print "Digest::SHA.pm Version           : ${Digest::SHA::VERSION}\n" if ($Digest::SHA::VERSION);
  97 -  print "Digest::SHA::PurePerl.pm Version : ${Digest::SHA::PurePerl::VERSION}\n" if ($Digest::SHA::PurePerl::VERSION);
  98 +  print "LWP::UserAgent.pm Version        : @{[$have_lwp ? ${LWP::UserAgent::VERSION} : 'n/a']}\n";
  99 +  print "LWP.pm Version                   : @{[$have_lwp ? ${LWP::VERSION} : 'n/a']}\n";
 100 +  print "Digest::SHA.pm Version           : @{[${Digest::SHA::VERSION} ? ${Digest::SHA::VERSION} : 'n/a']}\n";
 101 +  print "Digest::SHA1.pm Version          : @{[${Digest::SHA1::VERSION} ? ${Digest::SHA1::VERSION} : 'n/a']}\n";
 102 +  print "Digest::SHA::PurePerl.pm Version : @{[${Digest::SHA::PurePerl::VERSION} ? ${Digest::SHA::PurePerl::VERSION} : 'n/a']}\n";
 103 +  print "cURL Version                     : @{[$curl =~ /^curl (\d\.\d+(?:\.\d+)*)/ ? $1 : 'n/a']}\n";
 104    print ("=" x 78 . "\n");
 105  }
 106
 107 @@ -149,7 +171,7 @@ sub warning_message() {
 108    if ( $opt_d =~ m/^risk$/i ) { # Long Form Warning and Exit
 109      print "Warning: Use of this script may pose some risk:\n";
 110      print "\n";
 111 -    print "  1) If you use HTTP URLs they are subject to a man in the middle attack\n";
 112 +    print "  1) If you use http: URLs they are subject to a man in the middle attack\n";
 113      print "  2) Default to 'release', but more recent updates may be found in other trees\n";
 114      print "  3) certdata.txt file format may change, lag time to update this script\n";
 115      print "  4) Generally unwise to blindly trust CAs without manual review & verification\n";
 116 @@ -158,7 +180,7 @@ sub warning_message() {
 117      print "     swear at you.  ;)\n";
 118      exit;
 119    } else { # Short Form Warning
 120 -    print "Warning: Use of this script may pose some risk, -d risk for more details.\n";
 121 +    print STDERR "Warning: Use of this script may pose some risk, -d risk for more details.\n";
 122    }
 123  }
 124
 125 @@ -168,12 +190,13 @@ sub HELP_MESSAGE() {
 126    print "\t-d\tspecify Mozilla tree to pull certdata.txt or custom URL\n";
 127    print "\t\t  Valid names are:\n";
 128    print "\t\t    ", join( ", ", map { ( $_ =~ m/$opt_d/ ) ? "$_ (default)" : "$_" } sort keys %urls ), "\n";
 129 +  print "\t\t    ", join( "\n\t\t    ", map { sprintf "%-8s %s", "$_:", $urls{$_} } sort keys %urls ), "\n" if $opt_v;
 130    print "\t-f\tforce rebuild even if certdata.txt is current\n";
 131    print "\t-i\tprint version info about used modules\n";
 132 -  print "\t-k\tallow URLs other than HTTPS, enable HTTP fallback (insecure)\n";
 133 +  print "\t-k\tallow URLs other than https:, enable http: fallback (insecure)\n";
 134    print "\t-l\tprint license info about certdata.txt\n";
 135    print "\t-m\tinclude meta data in output\n";
 136 -  print "\t-n\tno download of certdata.txt (to use existing)\n";
 137 +  print wrap("\t","\t\t", "-n\tno download of certdata.txt (to use existing)@{[$curl||$have_lwp?'':' -- required on this OS as neither curl nor LWP::UserAgent is present']}"), "\n";
 138    print wrap("\t","\t\t", "-p\tlist of Mozilla trust purposes and levels for certificates to include in output. Takes the form of a comma separated list of purposes, a colon, and a comma separated list of levels. (default: $default_mozilla_trust_purposes:$default_mozilla_trust_levels)"), "\n";
 139    print "\t\t  Valid purposes are:\n";
 140    print wrap("\t\t    ","\t\t    ", join( ", ", "ALL", @valid_mozilla_trust_purposes ) ), "\n";
 141 @@ -185,7 +208,7 @@ sub HELP_MESSAGE() {
 142    print wrap("\t\t    ","\t\t    ", join( ", ", "ALL", @valid_signature_algorithms ) ), "\n";
 143    print "\t-t\tinclude plain text listing of certificates\n";
 144    print "\t-u\tunlink (remove) certdata.txt after processing\n";
 145 -  print "\t-v\tbe verbose and print out processed CAs\n";
 146 +  print "\t-v\tbe verbose and print out processed CAs and include URLs in help output\n";
 147    print "\t-w <l>\twrap base64 output lines after <l> chars (default: ${opt_w})\n";
 148    exit;
 149  }
 150 @@ -236,34 +259,36 @@ sub parse_csv_param($$@) {
 151    return @values;
 152  }
 153
 154 -sub sha256 {
 155 +sub sha1 {
 156    my $result;
 157 -  if ($Digest::SHA::VERSION || $Digest::SHA::PurePerl::VERSION) {
 158 +  if ($have_sha1) {
 159      open(FILE, $_[0]) or die "Can't open '$_[0]': $!";
 160      binmode(FILE);
 161 -    $result = $MOD_SHA->new(256)->addfile(*FILE)->hexdigest;
 162 +    local $/ = undef;
 163 +    $result = sha1_hex(scalar(<FILE>));
 164      close(FILE);
 165    } else {
 166      # Use OpenSSL command if Perl Digest::SHA modules not available
 167 -    $result = `"$openssl" dgst -r -sha256 "$_[0]"`;
 168 -    $result =~ s/^([0-9a-f]{64}) .+/$1/is;
 169 +    chomp($result = qx("$openssl" dgst -sha1 <"$_[0]"));
 170 +    $result =~ s/^[^=]*= *//;
 171    }
 172    return $result;
 173  }
 174
 175 -
 176 -sub oldhash {
 177 -  my $hash = "";
 178 -  open(C, "<$_[0]") || return 0;
 179 -  while(<C>) {
 180 -    chomp;
 181 -    if($_ =~ /^\#\# SHA256: (.*)/) {
 182 -      $hash = $1;
 183 -      last;
 184 +sub oldsha1 {
 185 +    my ($crt)=@_;
 186 +    my $sha1="<no file present>";
 187 +    if (open(C, "<$crt")) {
 188 +      while(<C>) {
 189 +          chomp;
 190 +          if($_ =~ /^\#\# SHA1: (.*)/) {
 191 +              $sha1 = $1;
 192 +              last;
 193 +          }
 194 +      }
 195 +      close(C);
 196      }
 197 -  }
 198 -  close(C);
 199 -  return $hash;
 200 +    return $sha1;
 201  }
 202
 203  if ( $opt_p !~ m/:/ ) {
 204 @@ -295,73 +320,85 @@ my $stdout = $crt eq '-';
 205  my $resp;
 206  my $fetched;
 207
 208 -my $oldhash = oldhash($crt);
 209 +my $oldsha1= $stdout ? '' : oldsha1($crt);
 210
 211 -report "SHA256 of old file: $oldhash";
 212 +report "SHA1 of old data file: $oldsha1" unless $stdout;
 213
 214 -if(!$opt_n) {
 215 -  report "Downloading $txt ...";
 216 -
 217 -  # If we have an HTTPS URL then use curl
 218 -  if($url =~ /^https:\/\//i) {
 219 -    if($curl) {
 220 -      if($curl =~ /^Protocols:.* https( |$)/m) {
 221 -        report "Get certdata with curl!";
 222 -        my $proto = !$opt_k ? "--proto =https" : "";
 223 -        my $quiet = $opt_q ? "-s" : "";
 224 -        my @out = `curl -w %{response_code} $proto $quiet -o "$txt" "$url"`;
 225 -        if(@out && $out[0] == 200) {
 226 -          $fetched = 1;
 227 -          report "Downloaded $txt";
 228 -        }
 229 -        else {
 230 -          report "Failed downloading via HTTPS with curl";
 231 -          if(-e $txt && !unlink($txt)) {
 232 -            report "Failed to remove '$txt': $!";
 233 -          }
 234 +unless ($opt_n and -e $txt) {
 235 +  if ($opt_n) {
 236 +    print STDERR "No '$txt' file found to process and option -n given.\n";
 237 +    exit 1;
 238 +  }
 239 +  if (!$curl && !$have_lwp) {
 240 +    print STDERR
 241 +      "The -n option is required on this OS as neither curl nor LWP::UserAgent\n",
 242 +      "is present.  Use the -v and -h options together to see the source URLs,\n",
 243 +      "download a suitable certdata.txt file via other means (such as wget) and\n",
 244 +      "run this script again using the -n option to process the certdata.txt file.\n";
 245 +    exit 1;
 246 +  }
 247 +  report "Downloading '$txt' ...";
 248 +  if ($curl) {
 249 +    if($curl =~ /^Protocols:.* https( |$)/m) {
 250 +      my $https = $url;
 251 +      $https =~ s/^http:/https:/;
 252 +      report "Getting certdata over https: with curl!";
 253 +      my $proto = !$opt_k ? "--proto =https" : "";
 254 +      my $quiet = $opt_q ? "-s" : "";
 255 +      my @out = `curl -w %{response_code} $proto $quiet -o "$txt" "$https"`;
 256 +      if(@out && $out[0] == 200) {
 257 +        $fetched = 1;
 258 +        report "Downloaded $txt";
 259 +      } else {
 260 +        report "Failed downloading via https: with curl@{[$have_lwp ? ', trying http: with LWP' : '']}";
 261 +        if(-e $txt && !unlink($txt)) {
 262 +          report "Failed to remove '$txt': $!";
 263          }
 264        }
 265 -      else {
 266 -        report "curl lacks https support";
 267 -      }
 268      }
 269      else {
 270 -      report "curl not found";
 271 +      report "curl lacks https: support";
 272      }
 273    }
 274 +  else {
 275 +    report "curl not found";
 276 +  }
 277
 278 -  # If nothing was fetched then use LWP
 279 -  if(!$fetched) {
 280 +  if (!$fetched && $have_lwp) {
 281      if($url =~ /^https:\/\//i) {
 282 -      report "Falling back to HTTP";
 283 +      report "Falling back to http:";
 284        $url =~ s/^https:\/\//http:\/\//i;
 285      }
 286      if(!$opt_k) {
 287 -      report "URLs other than HTTPS are disabled by default, to enable use -k";
 288 -      exit 1;
 289 -    }
 290 -    report "Get certdata with LWP!";
 291 -    if(!defined(${LWP::UserAgent::VERSION})) {
 292 -      report "LWP is not available (LWP::UserAgent not found)";
 293 +      report "URLs other than https: are disabled by default, to enable use -k";
 294        exit 1;
 295      }
 296 -    my $ua  = new LWP::UserAgent(agent => "$0/$version");
 297 +    report "Getting certdata over http: with LWP::UserAgent";
 298 +    my $ua = new LWP::UserAgent(agent => "$0/$version");
 299      $ua->env_proxy();
 300      $resp = $ua->mirror($url, $txt);
 301 -    if($resp && $resp->code eq '304') {
 302 +    if ($resp && $resp->code eq '304') {
 303        report "Not modified";
 304        exit 0 if -e $crt && !$opt_f;
 305 -    }
 306 -    else {
 307 -      $fetched = 1;
 308 -      report "Downloaded $txt";
 309 +    } else {
 310 +       $fetched = 1;
 311 +        report "Downloaded $txt";
 312      }
 313      if(!$resp || $resp->code !~ /^(?:200|304)$/) {
 314 -      report "Unable to download latest data: "
 315 -        . ($resp? $resp->code . ' - ' . $resp->message : "LWP failed");
 316 -      exit 1 if -e $crt || ! -r $txt;
 317 +       report "Unable to download latest data: "
 318 +         . ($resp? $resp->code . ' - ' . $resp->message : "LWP failed");
 319 +       exit 1 if -e $crt || ! -r $txt;
 320      }
 321    }
 322 +
 323 +  unless ($fetched) {
 324 +    print STDERR
 325 +      "Failed to download '$txt'.\n",
 326 +      "Please try again or use the -v and -h options together to see the source\n",
 327 +      "URLs, download a suitable certdata.txt file via other means (such as wget)\n",
 328 +      "and run this script again using the -n option to process the certdata.txt file.\n";
 329 +    exit 1;
 330 +  }
 331  }
 332
 333  my $filedate = $resp ? $resp->last_modified : (stat($txt))[9];
 334 @@ -373,14 +410,14 @@ if(!$filedate) {
 335  }
 336
 337  # get the hash from the download file
 338 -my $newhash= sha256($txt);
 339 +my $newsha1= sha1($txt);
 340
 341 -if(!$opt_f && $oldhash eq $newhash) {
 342 +if(!$opt_f && $oldsha1 eq $newsha1) {
 343      report "Downloaded file identical to previous run\'s source file. Exiting";
 344      exit;
 345  }
 346
 347 -report "SHA256 of new file: $newhash";
 348 +report "SHA1 of new data file: $newsha1";
 349
 350  my $currentdate = scalar gmtime($filedate);
 351
 352 @@ -407,7 +444,7 @@ print CRT <<EOT;
 353  ## Just configure this file as the SSLCACertificateFile.
 354  ##
 355  ## Conversion done with mk-ca-bundle.pl version $version.
 356 -## SHA256: $newhash
 357 +## SHA1: $newsha1
 358  ##
 359
 360  EOT
 361 @@ -488,7 +525,7 @@ while (<TXT>) {
 362                . "-----END CERTIFICATE-----\n";
 363        print CRT "\n$caname\n";
 364        print CRT @precert if($opt_m);
 365 -      my $maxStringLength = length(decode('UTF-8', $caname, Encode::FB_CROAK));
 366 +      my $maxStringLength = length(decode('UTF-8', $caname, Encode::FB_DEFAULT));
 367        if ($opt_t) {
 368          foreach my $key (keys %trust_purposes_by_level) {
 369             my $string = $key . ": " . join(", ", @{$trust_purposes_by_level{$key}});
 370 ---