src/uniq.c

   1 /* uniq -- remove duplicate lines from a sorted file
   2    Copyright (C) 86, 91, 1995-2002, Free Software Foundation, Inc.
   3
   4    This program is free software; you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation; either version 2, or (at your option)
   7    any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program; if not, write to the Free Software Foundation,
  16    Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
  17
  18 /* Written by Richard Stallman and David MacKenzie. */
  19 \f
  20 #include <config.h>
  21
  22 #include <stdio.h>
  23 #include <getopt.h>
  24 #include <sys/types.h>
  25
  26 #include "system.h"
  27 #include "closeout.h"
  28 #include "argmatch.h"
  29 #include "linebuffer.h"
  30 #include "error.h"
  31 #include "hard-locale.h"
  32 #include "posixver.h"
  33 #include "xmemcoll.h"
  34 #include "xstrtol.h"
  35 #include "memcasecmp.h"
  36
  37 /* The official name of this program (e.g., no `g' prefix).  */
  38 #define PROGRAM_NAME "uniq"
  39
  40 #define AUTHORS N_ ("Richard Stallman and David MacKenzie")
  41
  42 #define SWAP_LINES(A, B)                        \
  43   do                                            \
  44     {                                           \
  45       struct linebuffer *_tmp;                  \
  46       _tmp = (A);                               \
  47       (A) = (B);                                \
  48       (B) = _tmp;                               \
  49     }                                           \
  50   while (0)
  51
  52 /* The name this program was run with. */
  53 char *program_name;
  54
  55 /* Nonzero if the LC_COLLATE locale is hard.  */
  56 static int hard_LC_COLLATE;
  57
  58 /* Number of fields to skip on each line when doing comparisons. */
  59 static size_t skip_fields;
  60
  61 /* Number of chars to skip after skipping any fields. */
  62 static size_t skip_chars;
  63
  64 /* Number of chars to compare. */
  65 static size_t check_chars;
  66
  67 enum countmode
  68 {
  69   count_occurrences,            /* -c Print count before output lines. */
  70   count_none                    /* Default.  Do not print counts. */
  71 };
  72
  73 /* Whether and how to precede the output lines with a count of the number of
  74    times they occurred in the input. */
  75 static enum countmode countmode;
  76
  77 enum output_mode
  78 {
  79   output_repeated,              /* -d Only lines that are repeated. */
  80   output_all_repeated,          /* -D All lines that are repeated. */
  81   output_unique,                /* -u Only lines that are not repeated. */
  82   output_all                    /* Default.  Print first copy of each line. */
  83 };
  84
  85 /* Which lines to output. */
  86 static enum output_mode mode;
  87
  88 /* If nonzero, ignore case when comparing.  */
  89 static int ignore_case;
  90
  91 enum delimit_method
  92 {
  93   /* No delimiters output.  --all-repeated[=none] */
  94   DM_NONE,
  95
  96   /* Delimiter precedes all groups.  --all-repeated=prepend */
  97   DM_PREPEND,
  98
  99   /* Delimit all groups.  --all-repeated=separate */
 100   DM_SEPARATE
 101 };
 102
 103 static char const *const delimit_method_string[] =
 104 {
 105   "none", "prepend", "separate", 0
 106 };
 107
 108 static enum delimit_method const delimit_method_map[] =
 109 {
 110   DM_NONE, DM_PREPEND, DM_SEPARATE
 111 };
 112
 113 /* Select whether/how to delimit groups of duplicate lines.  */
 114 static enum delimit_method delimit_groups;
 115
 116 static struct option const longopts[] =
 117 {
 118   {"count", no_argument, NULL, 'c'},
 119   {"repeated", no_argument, NULL, 'd'},
 120   {"all-repeated", optional_argument, NULL, 'D'},
 121   {"ignore-case", no_argument, NULL, 'i'},
 122   {"unique", no_argument, NULL, 'u'},
 123   {"skip-fields", required_argument, NULL, 'f'},
 124   {"skip-chars", required_argument, NULL, 's'},
 125   {"check-chars", required_argument, NULL, 'w'},
 126   {GETOPT_HELP_OPTION_DECL},
 127   {GETOPT_VERSION_OPTION_DECL},
 128   {NULL, 0, NULL, 0}
 129 };
 130
 131 void
 132 usage (int status)
 133 {
 134   if (status != 0)
 135     fprintf (stderr, _("Try `%s --help' for more information.\n"),
 136              program_name);
 137   else
 138     {
 139       printf (_("\
 140 Usage: %s [OPTION]... [INPUT [OUTPUT]]\n\
 141 "),
 142               program_name);
 143       fputs (_("\
 144 Discard all but one of successive identical lines from INPUT (or\n\
 145 standard input), writing to OUTPUT (or standard output).\n\
 146 \n\
 147 "), stdout);
 148      fputs (_("\
 149 Mandatory arguments to long options are mandatory for short options too.\n\
 150 "), stdout);
 151      fputs (_("\
 152   -c, --count           prefix lines by the number of occurrences\n\
 153   -d, --repeated        only print duplicate lines\n\
 154 "), stdout);
 155      fputs (_("\
 156   -D, --all-repeated[=delimit-method] print all duplicate lines\n\
 157                         delimit-method={none(default),prepend,separate}\n\
 158                         Delimiting is done with blank lines.\n\
 159   -f, --skip-fields=N   avoid comparing the first N fields\n\
 160   -i, --ignore-case     ignore differences in case when comparing\n\
 161   -s, --skip-chars=N    avoid comparing the first N characters\n\
 162   -u, --unique          only print unique lines\n\
 163 "), stdout);
 164      fputs (_("\
 165   -w, --check-chars=N   compare no more than N characters in lines\n\
 166 "), stdout);
 167      fputs (HELP_OPTION_DESCRIPTION, stdout);
 168      fputs (VERSION_OPTION_DESCRIPTION, stdout);
 169      fputs (_("\
 170 \n\
 171 A field is a run of whitespace, then non-whitespace characters.\n\
 172 Fields are skipped before chars.\n\
 173 "), stdout);
 174       printf (_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
 175     }
 176   exit (status == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
 177 }
 178
 179 /* Convert OPT to size_t, reporting an error using MSGID if it does
 180    not fit.  */
 181
 182 static size_t
 183 size_opt (char const *opt, char const *msgid)
 184 {
 185   unsigned long int size;
 186   if (xstrtoul (opt, NULL, 10, &size, "") != LONGINT_OK
 187       || SIZE_MAX < size)
 188     error (EXIT_FAILURE, 0, "%s: %s", opt, _(msgid));
 189   return size;
 190 }
 191
 192 /* Given a linebuffer LINE,
 193    return a pointer to the beginning of the line's field to be compared. */
 194
 195 static char *
 196 find_field (const struct linebuffer *line)
 197 {
 198   register size_t count;
 199   register char *lp = line->buffer;
 200   register size_t size = line->length - 1;
 201   register size_t i = 0;
 202
 203   for (count = 0; count < skip_fields && i < size; count++)
 204     {
 205       while (i < size && ISBLANK (lp[i]))
 206         i++;
 207       while (i < size && !ISBLANK (lp[i]))
 208         i++;
 209     }
 210
 211   for (count = 0; count < skip_chars && i < size; count++)
 212     i++;
 213
 214   return lp + i;
 215 }
 216
 217 /* Return zero if two strings OLD and NEW match, nonzero if not.
 218    OLD and NEW point not to the beginnings of the lines
 219    but rather to the beginnings of the fields to compare.
 220    OLDLEN and NEWLEN are their lengths. */
 221
 222 static int
 223 different (char *old, char *new, size_t oldlen, size_t newlen)
 224 {
 225   if (check_chars < oldlen)
 226     oldlen = check_chars;
 227   if (check_chars < newlen)
 228     newlen = check_chars;
 229
 230   if (ignore_case)
 231     {
 232       /* FIXME: This should invoke strcoll somehow.  */
 233       return oldlen != newlen || memcasecmp (old, new, oldlen);
 234     }
 235   else if (HAVE_SETLOCALE && hard_LC_COLLATE)
 236     return xmemcoll (old, oldlen, new, newlen);
 237   else
 238     return oldlen != newlen || memcmp (old, new, oldlen);
 239 }
 240
 241 /* Output the line in linebuffer LINE to stream STREAM
 242    provided that the switches say it should be output.
 243    If requested, print the number of times it occurred, as well;
 244    LINECOUNT + 1 is the number of times that the line occurred. */
 245
 246 static void
 247 writeline (const struct linebuffer *line, FILE *stream, int linecount)
 248 {
 249   if ((mode == output_unique && linecount != 0)
 250       || (mode == output_repeated && linecount == 0)
 251       || (mode == output_all_repeated && linecount == 0))
 252     return;
 253
 254   if (countmode == count_occurrences)
 255     fprintf (stream, "%7d\t", linecount + 1);
 256
 257   fwrite (line->buffer, sizeof (char), line->length, stream);
 258 }
 259
 260 /* Process input file INFILE with output to OUTFILE.
 261    If either is "-", use the standard I/O stream for it instead. */
 262
 263 static void
 264 check_file (const char *infile, const char *outfile)
 265 {
 266   FILE *istream;
 267   FILE *ostream;
 268   struct linebuffer lb1, lb2;
 269   struct linebuffer *thisline, *prevline;
 270
 271   if (STREQ (infile, "-"))
 272     istream = stdin;
 273   else
 274     istream = fopen (infile, "r");
 275   if (istream == NULL)
 276     error (EXIT_FAILURE, errno, "%s", infile);
 277
 278   if (STREQ (outfile, "-"))
 279     ostream = stdout;
 280   else
 281     ostream = fopen (outfile, "w");
 282   if (ostream == NULL)
 283     error (EXIT_FAILURE, errno, "%s", outfile);
 284
 285   thisline = &lb1;
 286   prevline = &lb2;
 287
 288   initbuffer (thisline);
 289   initbuffer (prevline);
 290
 291   /* The duplication in the following `if' and `else' blocks is an
 292      optimization to distinguish the common case (in which none of
 293      the following options has been specified: --count, -repeated,
 294      --all-repeated, --unique) from the others.  In the common case,
 295      this optimization lets uniq output each different line right away,
 296      without waiting to see if the next one is different.  */
 297
 298   if (mode == output_all && countmode == count_none)
 299     {
 300       char *prevfield IF_LINT (= NULL);
 301       size_t prevlen IF_LINT (= 0);
 302
 303       while (!feof (istream))
 304         {
 305           char *thisfield;
 306           size_t thislen;
 307           if (readline (thisline, istream) == 0)
 308             break;
 309           thisfield = find_field (thisline);
 310           thislen = thisline->length - 1 - (thisfield - thisline->buffer);
 311           if (prevline->length == 0
 312               || different (thisfield, prevfield, thislen, prevlen))
 313             {
 314               fwrite (thisline->buffer, sizeof (char),
 315                       thisline->length, ostream);
 316
 317               SWAP_LINES (prevline, thisline);
 318               prevfield = thisfield;
 319               prevlen = thislen;
 320             }
 321         }
 322     }
 323   else
 324     {
 325       char *prevfield;
 326       size_t prevlen;
 327       int match_count = 0;
 328       int first_delimiter = 1;
 329
 330       if (readline (prevline, istream) == 0)
 331         goto closefiles;
 332       prevfield = find_field (prevline);
 333       prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
 334
 335       while (!feof (istream))
 336         {
 337           int match;
 338           char *thisfield;
 339           size_t thislen;
 340           if (readline (thisline, istream) == 0)
 341             break;
 342           thisfield = find_field (thisline);
 343           thislen = thisline->length - 1 - (thisfield - thisline->buffer);
 344           match = !different (thisfield, prevfield, thislen, prevlen);
 345
 346           if (match)
 347             ++match_count;
 348
 349           if (mode == output_all_repeated && delimit_groups != DM_NONE)
 350             {
 351               if (!match)
 352                 {
 353                   if (match_count) /* a previous match */
 354                     first_delimiter = 0; /* Only used when DM_SEPARATE */
 355                 }
 356               else if (match_count == 1)
 357                 {
 358                   if ((delimit_groups == DM_PREPEND)
 359                       || (delimit_groups == DM_SEPARATE
 360                           && !first_delimiter))
 361                     putc ('\n', ostream);
 362                 }
 363             }
 364
 365           if (!match || mode == output_all_repeated)
 366             {
 367               writeline (prevline, ostream, match_count);
 368               SWAP_LINES (prevline, thisline);
 369               prevfield = thisfield;
 370               prevlen = thislen;
 371               if (!match)
 372                 match_count = 0;
 373             }
 374         }
 375
 376       writeline (prevline, ostream, match_count);
 377     }
 378
 379  closefiles:
 380   if (ferror (istream) || fclose (istream) == EOF)
 381     error (EXIT_FAILURE, errno, _("error reading %s"), infile);
 382
 383   /* Close ostream only if it's not stdout -- the latter is closed
 384      via the atexit-invoked close_stdout.  */
 385   if (ostream != stdout && (ferror (ostream) || fclose (ostream) == EOF))
 386     error (EXIT_FAILURE, errno, _("error writing %s"), outfile);
 387
 388   free (lb1.buffer);
 389   free (lb2.buffer);
 390 }
 391
 392 int
 393 main (int argc, char **argv)
 394 {
 395   int optc = 0;
 396   bool posixly_correct = (getenv ("POSIXLY_CORRECT") != NULL);
 397   bool obsolete_skip_fields = false;
 398   int nfiles = 0;
 399   char const *file[2];
 400
 401   file[0] = file[1] = "-";
 402   program_name = argv[0];
 403   setlocale (LC_ALL, "");
 404   bindtextdomain (PACKAGE, LOCALEDIR);
 405   textdomain (PACKAGE);
 406   hard_LC_COLLATE = hard_locale (LC_COLLATE);
 407
 408   atexit (close_stdout);
 409
 410   skip_chars = 0;
 411   skip_fields = 0;
 412   check_chars = SIZE_MAX;
 413   mode = output_all;
 414   countmode = count_none;
 415   delimit_groups = DM_NONE;
 416
 417   for (;;)
 418     {
 419       /* Parse an operand with leading "+" as a file after "--" was
 420          seen; or if pedantic and a file was seen; or if not
 421          obsolete.  */
 422
 423       if (optc == -1
 424           || (posixly_correct && nfiles != 0)
 425           || ((optc = getopt_long (argc, argv,
 426                                    "-0123456789Dcdf:is:uw:", longopts, NULL))
 427               == -1))
 428         {
 429           if (optind == argc)
 430             break;
 431           if (nfiles == 2)
 432             {
 433               error (0, 0, _("extra operand `%s'"), argv[optind]);
 434               usage (EXIT_FAILURE);
 435             }
 436           file[nfiles++] = argv[optind++];
 437         }
 438       else switch (optc)
 439         {
 440         case 1:
 441           {
 442             unsigned long int size;
 443             if (optarg[0] == '+'
 444                 && posix2_version () < 200112
 445                 && xstrtoul (optarg, NULL, 10, &size, "") == LONGINT_OK
 446                 && size <= SIZE_MAX)
 447               skip_chars = size;
 448             else if (nfiles == 2)
 449               {
 450                 error (0, 0, _("extra operand `%s'"), optarg);
 451                 usage (EXIT_FAILURE);
 452               }
 453             else
 454               file[nfiles++] = optarg;
 455           }
 456           break;
 457
 458         case '0':
 459         case '1':
 460         case '2':
 461         case '3':
 462         case '4':
 463         case '5':
 464         case '6':
 465         case '7':
 466         case '8':
 467         case '9':
 468           {
 469             size_t s = skip_fields;
 470             skip_fields = s * 10 + optc - '0';
 471             if (SIZE_MAX / 10 < s || skip_fields < s)
 472               error (EXIT_FAILURE, 0, "%s",
 473                      _("invalid number of fields to skip"));
 474             obsolete_skip_fields = true;
 475           }
 476           break;
 477
 478         case 'c':
 479           countmode = count_occurrences;
 480           break;
 481
 482         case 'd':
 483           mode = output_repeated;
 484           break;
 485
 486         case 'D':
 487           mode = output_all_repeated;
 488           if (optarg == NULL)
 489             delimit_groups = DM_NONE;
 490           else
 491             delimit_groups = XARGMATCH ("--all-repeated", optarg,
 492                                         delimit_method_string,
 493                                         delimit_method_map);
 494           break;
 495
 496         case 'f':               /* Like '-#'. */
 497           skip_fields = size_opt (optarg,
 498                                   N_("invalid number of fields to skip"));
 499           break;
 500
 501         case 'i':
 502           ignore_case = 1;
 503           break;
 504
 505         case 's':               /* Like '+#'. */
 506           skip_chars = size_opt (optarg,
 507                                  N_("invalid number of bytes to skip"));
 508           break;
 509
 510         case 'u':
 511           mode = output_unique;
 512           break;
 513
 514         case 'w':
 515           check_chars = size_opt (optarg,
 516                                   N_("invalid number of bytes to compare"));
 517           break;
 518
 519         case_GETOPT_HELP_CHAR;
 520
 521         case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
 522
 523         default:
 524           usage (EXIT_FAILURE);
 525         }
 526     }
 527
 528   if (obsolete_skip_fields && 200112 <= posix2_version ())
 529     {
 530       error (0, 0, _("`-%lu' option is obsolete; use `-f %lu'"),
 531              (unsigned long) skip_fields, (unsigned long) skip_fields);
 532       usage (EXIT_FAILURE);
 533     }
 534
 535   if (countmode == count_occurrences && mode == output_all_repeated)
 536     {
 537       error (0, 0,
 538            _("printing all duplicated lines and repeat counts is meaningless"));
 539       usage (EXIT_FAILURE);
 540     }
 541
 542   check_file (file[0], file[1]);
 543
 544   exit (EXIT_SUCCESS);
 545 }