intl/chardet/tools/charfreqtostat.pl

   1 #!/usr/bin/perl
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is mozilla.org code.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # Netscape Communications Corporation.
  20 # Portions created by the Initial Developer are Copyright (C) 1999
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #
  25 # Alternatively, the contents of this file may be used under the terms of
  26 # either the GNU General Public License Version 2 or later (the "GPL"), or
  27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28 # in which case the provisions of the GPL or the LGPL are applicable instead
  29 # of those above. If you wish to allow use of your version of this file only
  30 # under the terms of either the GPL or the LGPL, and not to allow others to
  31 # use your version of this file under the terms of the MPL, indicate your
  32 # decision by deleting the provisions above and replace them with the notice
  33 # and other provisions required by the GPL or the LGPL. If you do not delete
  34 # the provisions above, a recipient may use your version of this file under
  35 # the terms of any one of the MPL, the GPL or the LGPL.
  36 #
  37 # ***** END LICENSE BLOCK *****
  38 sub GenNPL {
  39   my($ret) = << "END_NPL";
  40 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  41 /* ***** BEGIN LICENSE BLOCK *****
  42  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  43  *
  44  * The contents of this file are subject to the Mozilla Public License Version
  45  * 1.1 (the "License"); you may not use this file except in compliance with
  46  * the License. You may obtain a copy of the License at
  47  * http://www.mozilla.org/MPL/
  48  *
  49  * Software distributed under the License is distributed on an "AS IS" basis,
  50  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  51  * for the specific language governing rights and limitations under the
  52  * License.
  53  *
  54  * The Original Code is mozilla.org code.
  55  *
  56  * The Initial Developer of the Original Code is
  57  * Netscape Communications Corporation.
  58  * Portions created by the Initial Developer are Copyright (C) 1998
  59  * the Initial Developer. All Rights Reserved.
  60  *
  61  * Contributor(s):
  62  *
  63  * Alternatively, the contents of this file may be used under the terms of
  64  * either the GNU General Public License Version 2 or later (the "GPL"), or
  65  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  66  * in which case the provisions of the GPL or the LGPL are applicable instead
  67  * of those above. If you wish to allow use of your version of this file only
  68  * under the terms of either the GPL or the LGPL, and not to allow others to
  69  * use your version of this file under the terms of the MPL, indicate your
  70  * decision by deleting the provisions above and replace them with the notice
  71  * and other provisions required by the GPL or the LGPL. If you do not delete
  72  * the provisions above, a recipient may use your version of this file under
  73  * the terms of any one of the MPL, the GPL or the LGPL.
  74  *
  75  * ***** END LICENSE BLOCK ***** */
  76 END_NPL
  77
  78   return $ret;
  79 }
  80
  81 print GenNPL();
  82 $total=0;
  83 @h;
  84 @l;
  85
  86 while(<STDIN>)
  87 {
  88    @k = split(/\s+/, $_);
  89   @i = unpack("CCCC", $k[0]);
  90 #  printf("%x %x %s",$i[0] ,  $i[1] , "[" . $k[0] . "]   " . $i . "  " . $j . "  " . $k[1]  ."\n");
  91   if((0xA1 <= $i[0]) && (0xA1 <= $i[1])){
  92   $total += $k[1];
  93      $v = $i[0] - 0x00A1;
  94      $h[$v] += $k[1];
  95      $u = $i[1] - 0x00A1;
  96      $l[$u] += $k[1];
  97 #     print "hello $v $h[$v] $u $l[$u]\n";
  98   }
  99 }
 100
 101
 102 $ffh = 0.0;
 103 $ffl = 0.0;
 104 for($i=0x00A1;$i< 0x00FF ; $i++)
 105 {
 106      $fh[$i - 0x00a1] = $h[$i- 0x00a1] / $total;
 107      $ffh += $fh[$i - 0x00a1];
 108
 109      $fl[$i - 0x00a1] = $l[$i- 0x00a1] / $total;
 110      $ffl += $fl[$i - 0x00a1];
 111 }
 112 $mh = $ffh / 94.0;
 113 $ml = $ffl / 94.0;
 114
 115 $sumh=0.0;
 116 $suml=0.0;
 117 for($i=0x00A1;$i< 0x00FF ; $i++)
 118 {
 119      $sh = $fh[$i - 0x00a1] - $mh;
 120      $sh *= $sh;
 121      $sumh += $sh;
 122
 123      $sl = $fl[$i - 0x00a1] - $ml;
 124      $sl *= $sl;
 125      $suml += $sl;
 126 }
 127 $sumh /= 94.0;
 128 $suml /= 94.0;
 129 $stdh = sqrt($sumh);
 130 $stdl = sqrt($suml);
 131
 132 print "{\n";
 133 print "  {\n";
 134 for($i=0x00A1;$i< 0x00FF ; $i++)
 135 {
 136    if($i eq 0xfe) {
 137      printf("   %.6ff  \/\/ FreqH[%2x]\n", $fh[$i - 0x00a1] , $i);
 138    } else {
 139      printf("   %.6ff, \/\/ FreqH[%2x]\n", $fh[$i - 0x00a1] , $i);
 140    }
 141 }
 142 print "  },\n";
 143 printf ("%.6ff, \/\/ Lead Byte StdDev\n", $stdh);
 144 printf ("%.6ff, \/\/ Lead Byte Mean\n", $mh);
 145 printf ("%.6ff, \/\/ Lead Byte Weight\n", $stdh / ($stdh + $stdl));
 146 print "  {\n";
 147 for($i=0x00A1;$i< 0x00FF ; $i++)
 148 {
 149    if($i eq 0xfe) {
 150      printf("  %.6ff  \/\/ FreqL[%2x]\n", $fl[$i - 0x00a1] ,  $i);
 151    } else {
 152      printf("  %.6ff, \/\/ FreqL[%2x]\n", $fl[$i - 0x00a1] ,  $i);
 153    }
 154 }
 155 print "  },\n";
 156 printf ("%.6ff, \/\/ Trail Byte StdDev\n", $stdl);
 157 printf ("%.6ff, \/\/ Trail Byte Mean\n", $ml);
 158 printf ("%.6ff  \/\/ Trial Byte Weight\n", $stdl / ($stdh + $stdl));
 159 print "};\n";