src/uniq.c

   1 /* uniq -- remove duplicate lines from a sorted file
   2    Copyright (C) 86, 91, 1995-1998, 1999 Free Software Foundation, Inc.
   3
   4    This program is free software; you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation; either version 2, or (at your option)
   7    any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program; if not, write to the Free Software Foundation,
  16    Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
  17
  18 /* Written by Richard Stallman and David MacKenzie. */
  19 \f
  20 #include <config.h>
  21
  22 #include <stdio.h>
  23 #include <getopt.h>
  24 #include <sys/types.h>
  25
  26 #include "system.h"
  27 #include "linebuffer.h"
  28 #include "error.h"
  29 #include "xstrtol.h"
  30 #include "memcasecmp.h"
  31
  32 /* The official name of this program (e.g., no `g' prefix).  */
  33 #define PROGRAM_NAME "uniq"
  34
  35 #define AUTHORS "Richard Stallman and David MacKenzie"
  36
  37 /* Undefine, to avoid warning about redefinition on some systems.  */
  38 #undef min
  39 #define min(x, y) ((x) < (y) ? (x) : (y))
  40
  41 /* The name this program was run with. */
  42 char *program_name;
  43
  44 /* Number of fields to skip on each line when doing comparisons. */
  45 static int skip_fields;
  46
  47 /* Number of chars to skip after skipping any fields. */
  48 static int skip_chars;
  49
  50 /* Number of chars to compare; if 0, compare the whole lines. */
  51 static int check_chars;
  52
  53 enum countmode
  54 {
  55   count_occurrences,            /* -c Print count before output lines. */
  56   count_none                    /* Default.  Do not print counts. */
  57 };
  58
  59 /* Whether and how to precede the output lines with a count of the number of
  60    times they occurred in the input. */
  61 static enum countmode countmode;
  62
  63 enum output_mode
  64 {
  65   output_repeated,              /* -d Only lines that are repeated. */
  66   output_all_repeated,          /* -D All lines that are repeated. */
  67   output_unique,                /* -u Only lines that are not repeated. */
  68   output_all                    /* Default.  Print first copy of each line. */
  69 };
  70
  71 /* Which lines to output. */
  72 static enum output_mode mode;
  73
  74 /* If nonzero, ignore case when comparing.  */
  75 static int ignore_case;
  76
  77 static struct option const longopts[] =
  78 {
  79   {"count", no_argument, NULL, 'c'},
  80   {"repeated", no_argument, NULL, 'd'},
  81   {"all-repeated", no_argument, NULL, 'D'},
  82   {"ignore-case", no_argument, NULL, 'i'},
  83   {"unique", no_argument, NULL, 'u'},
  84   {"skip-fields", required_argument, NULL, 'f'},
  85   {"skip-chars", required_argument, NULL, 's'},
  86   {"check-chars", required_argument, NULL, 'w'},
  87   {GETOPT_HELP_OPTION_DECL},
  88   {GETOPT_VERSION_OPTION_DECL},
  89   {NULL, 0, NULL, 0}
  90 };
  91
  92 void
  93 usage (int status)
  94 {
  95   if (status != 0)
  96     fprintf (stderr, _("Try `%s --help' for more information.\n"),
  97              program_name);
  98   else
  99     {
 100       printf (_("\
 101 Usage: %s [OPTION]... [INPUT [OUTPUT]]\n\
 102 "),
 103               program_name);
 104       printf (_("\
 105 Discard all but one of successive identical lines from INPUT (or\n\
 106 standard input), writing to OUTPUT (or standard output).\n\
 107 \n\
 108   -c, --count           prefix lines by the number of occurrences\n\
 109   -d, --repeated        only print duplicate lines\n\
 110   -D, --all-repeated    print all duplicate lines\n\
 111   -f, --skip-fields=N   avoid comparing the first N fields\n\
 112   -i, --ignore-case     ignore differences in case when comparing\n\
 113   -s, --skip-chars=N    avoid comparing the first N characters\n\
 114   -u, --unique          only print unique lines\n\
 115   -w, --check-chars=N   compare no more than N characters in lines\n\
 116   -N                    same as -f N\n\
 117   +N                    same as -s N\n\
 118       --help            display this help and exit\n\
 119       --version         output version information and exit\n\
 120 \n\
 121 A field is a run of whitespace, then non-whitespace characters.\n\
 122 Fields are skipped before chars.\n\
 123 "));
 124       puts (_("\nReport bugs to <bug-textutils@gnu.org>."));
 125     }
 126   exit (status == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
 127 }
 128
 129 /* Given a linebuffer LINE,
 130    return a pointer to the beginning of the line's field to be compared. */
 131
 132 static char *
 133 find_field (const struct linebuffer *line)
 134 {
 135   register int count;
 136   register char *lp = line->buffer;
 137   register size_t size = line->length;
 138   register size_t i = 0;
 139
 140   for (count = 0; count < skip_fields && i < size; count++)
 141     {
 142       while (i < size && ISBLANK (lp[i]))
 143         i++;
 144       while (i < size && !ISBLANK (lp[i]))
 145         i++;
 146     }
 147
 148   for (count = 0; count < skip_chars && i < size; count++)
 149     i++;
 150
 151   return lp + i;
 152 }
 153
 154 /* Return zero if two strings OLD and NEW match, nonzero if not.
 155    OLD and NEW point not to the beginnings of the lines
 156    but rather to the beginnings of the fields to compare.
 157    OLDLEN and NEWLEN are their lengths. */
 158
 159 static int
 160 different (const char *old, const char *new, size_t oldlen, size_t newlen)
 161 {
 162   register int order;
 163
 164   if (check_chars)
 165     {
 166       if (oldlen > check_chars)
 167         oldlen = check_chars;
 168       if (newlen > check_chars)
 169         newlen = check_chars;
 170     }
 171
 172   /* Use an if-statement here rather than a function variable to
 173      avoid portability hassles of getting a non-conflicting declaration
 174      of memcmp.  */
 175   if (ignore_case)
 176     order = memcasecmp (old, new, min (oldlen, newlen));
 177   else
 178     order = memcmp (old, new, min (oldlen, newlen));
 179
 180   if (order == 0)
 181     return oldlen - newlen;
 182   return order;
 183 }
 184
 185 /* Output the line in linebuffer LINE to stream STREAM
 186    provided that the switches say it should be output.
 187    If requested, print the number of times it occurred, as well;
 188    LINECOUNT + 1 is the number of times that the line occurred. */
 189
 190 static void
 191 writeline (const struct linebuffer *line, FILE *stream, int linecount)
 192 {
 193   if ((mode == output_unique && linecount != 0)
 194       || (mode == output_repeated && linecount == 0)
 195       || (mode == output_all_repeated && linecount == 0))
 196     return;
 197
 198   if (countmode == count_occurrences)
 199     fprintf (stream, "%7d\t", linecount + 1);
 200
 201   fwrite (line->buffer, sizeof (char), line->length, stream);
 202 }
 203
 204 /* Process input file INFILE with output to OUTFILE.
 205    If either is "-", use the standard I/O stream for it instead. */
 206
 207 static void
 208 check_file (const char *infile, const char *outfile)
 209 {
 210   FILE *istream;
 211   FILE *ostream;
 212   struct linebuffer lb1, lb2;
 213   struct linebuffer *thisline, *prevline, *exch;
 214   char *prevfield, *thisfield;
 215   size_t prevlen, thislen;
 216   int match_count = 0;
 217
 218   if (STREQ (infile, "-"))
 219     istream = stdin;
 220   else
 221     istream = fopen (infile, "r");
 222   if (istream == NULL)
 223     error (EXIT_FAILURE, errno, "%s", infile);
 224
 225   if (STREQ (outfile, "-"))
 226     ostream = stdout;
 227   else
 228     ostream = fopen (outfile, "w");
 229   if (ostream == NULL)
 230     error (EXIT_FAILURE, errno, "%s", outfile);
 231
 232   thisline = &lb1;
 233   prevline = &lb2;
 234
 235   initbuffer (thisline);
 236   initbuffer (prevline);
 237
 238   if (readline (prevline, istream) == 0)
 239     goto closefiles;
 240   prevfield = find_field (prevline);
 241   prevlen = prevline->length - (prevfield - prevline->buffer);
 242
 243   while (!feof (istream))
 244     {
 245       int match;
 246       if (readline (thisline, istream) == 0)
 247         break;
 248       thisfield = find_field (thisline);
 249       thislen = thisline->length - (thisfield - thisline->buffer);
 250       match = !different (thisfield, prevfield, thislen, prevlen);
 251
 252       if (match)
 253         ++match_count;
 254
 255       if (!match || mode == output_all_repeated)
 256         {
 257           writeline (prevline, ostream, match_count);
 258           exch = prevline;
 259           prevline = thisline;
 260           thisline = exch;
 261           prevfield = thisfield;
 262           prevlen = thislen;
 263           if (!match)
 264             match_count = 0;
 265         }
 266     }
 267
 268   writeline (prevline, ostream, match_count);
 269
 270  closefiles:
 271   if (ferror (istream) || fclose (istream) == EOF)
 272     error (EXIT_FAILURE, errno, _("error reading %s"), infile);
 273
 274   if (ferror (ostream) || fclose (ostream) == EOF)
 275     error (EXIT_FAILURE, errno, _("error writing %s"), outfile);
 276
 277   free (lb1.buffer);
 278   free (lb2.buffer);
 279 }
 280
 281 int
 282 main (int argc, char **argv)
 283 {
 284   int optc;
 285   char *infile = "-", *outfile = "-";
 286
 287   program_name = argv[0];
 288   setlocale (LC_ALL, "");
 289   bindtextdomain (PACKAGE, LOCALEDIR);
 290   textdomain (PACKAGE);
 291
 292   skip_chars = 0;
 293   skip_fields = 0;
 294   check_chars = 0;
 295   mode = output_all;
 296   countmode = count_none;
 297
 298   while ((optc = getopt_long (argc, argv, "0123456789cdDf:is:uw:", longopts,
 299                               NULL)) != -1)
 300     {
 301       switch (optc)
 302         {
 303         case 0:
 304           break;
 305
 306         case '0':
 307         case '1':
 308         case '2':
 309         case '3':
 310         case '4':
 311         case '5':
 312         case '6':
 313         case '7':
 314         case '8':
 315         case '9':
 316           skip_fields = skip_fields * 10 + optc - '0';
 317           break;
 318
 319         case 'c':
 320           countmode = count_occurrences;
 321           break;
 322
 323         case 'd':
 324           mode = output_repeated;
 325           break;
 326
 327         case 'D':
 328           mode = output_all_repeated;
 329           break;
 330
 331         case 'f':               /* Like '-#'. */
 332           {
 333             long int tmp_long;
 334             if (xstrtol (optarg, NULL, 10, &tmp_long, "") != LONGINT_OK
 335                 || tmp_long <= 0 || tmp_long > INT_MAX)
 336               error (EXIT_FAILURE, 0,
 337                      _("invalid number of fields to skip: `%s'"),
 338                      optarg);
 339             skip_fields = (int) tmp_long;
 340           }
 341           break;
 342
 343         case 'i':
 344           ignore_case = 1;
 345           break;
 346
 347         case 's':               /* Like '+#'. */
 348           {
 349             long int tmp_long;
 350             if (xstrtol (optarg, NULL, 10, &tmp_long, "") != LONGINT_OK
 351                 || tmp_long <= 0 || tmp_long > INT_MAX)
 352               error (EXIT_FAILURE, 0,
 353                      _("invalid number of bytes to skip: `%s'"),
 354                      optarg);
 355             skip_chars = (int) tmp_long;
 356           }
 357           break;
 358
 359         case 'u':
 360           mode = output_unique;
 361           break;
 362
 363         case 'w':
 364           {
 365             long int tmp_long;
 366             if (xstrtol (optarg, NULL, 10, &tmp_long, "") != LONGINT_OK
 367                 || tmp_long <= 0 || tmp_long > INT_MAX)
 368               error (EXIT_FAILURE, 0,
 369                      _("invalid number of bytes to compare: `%s'"),
 370                      optarg);
 371             check_chars = (int) tmp_long;
 372           }
 373           break;
 374
 375         case_GETOPT_HELP_CHAR;
 376
 377         case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
 378
 379         default:
 380           usage (1);
 381         }
 382     }
 383
 384   if (optind >= 2 && !STREQ (argv[optind - 1], "--"))
 385     {
 386       /* Interpret non-option arguments with leading `+' only
 387          if we haven't seen `--'.  */
 388       while (optind < argc && argv[optind][0] == '+')
 389         {
 390           char *opt_str = argv[optind++];
 391           long int tmp_long;
 392           if (xstrtol (opt_str, NULL, 10, &tmp_long, "") != LONGINT_OK
 393               || tmp_long <= 0 || tmp_long > INT_MAX)
 394             error (EXIT_FAILURE, 0,
 395                    _("invalid number of bytes to compare: `%s'"),
 396                    opt_str);
 397           skip_chars = (int) tmp_long;
 398         }
 399     }
 400
 401   if (optind < argc)
 402     infile = argv[optind++];
 403
 404   if (optind < argc)
 405     outfile = argv[optind++];
 406
 407   if (optind < argc)
 408     {
 409       error (0, 0, _("too many arguments"));
 410       usage (1);
 411     }
 412
 413   if (countmode == count_occurrences && mode == output_all_repeated)
 414     {
 415       error (0, 0,
 416            _("printing all duplicated lines and repeat counts is meaningless"));
 417       usage (1);
 418     }
 419
 420   check_file (infile, outfile);
 421
 422   exit (EXIT_SUCCESS);
 423 }