intl/chardet/tools/charfreq.pl

   1 #!/usr/bin/perl
   2 #!/usr/bin/perl
   3 #
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is mozilla.org code.
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Netscape Communications Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 1999
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s):
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either of the GNU General Public License Version 2 or later (the "GPL"),
  28 # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39 open (STAT,$ARGV[0]) || die " cannot open data file $ARGV[0]\n";
  40 @count;
  41 while(<STAT>)
  42 {
  43    @k = split(/\s+/, $_);
  44    $count{$k[0]} = $k[1];
  45 }
  46 $count = 0;
  47 while(<STDIN>)
  48 {
  49   @ck = split /\s*/, $_;
  50   $s = 0;
  51   $fb = 0;
  52   $cl = $#ck;
  53   $j = 0;
  54   while($j < $cl) {
  55      $cc = unpack("C", $ck[$j]);
  56      if(0 eq $s ) {
  57        if($cc > 0x80) {
  58          if($cc > 0xa0) {
  59            $fb = $ck[$j];
  60            $s = 2;
  61          } else {
  62            $s = 1;
  63          }
  64        }
  65      } elsif (1 eq $s) {
  66      } else {
  67          if($cc > 0xa0) {
  68            $fb .= $ck[$j];
  69            $count{$fb}++;
  70            print $fb . " "  .$count{$fb} . "\n";
  71            $s = 0;
  72          } else {
  73            $s = 1;
  74          }
  75      }
  76      $j = $j + 1;
  77   }
  78 }
  79 foreach $c (sort(keys( %count )))
  80 {
  81    print $c . " ". $count{$c} . "\n";
  82 }