1 /* $Id: read.c,v 1.192 2017/07/20 14:36:36 schwarze Exp $ */
3 * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
4 * Copyright (c) 2010-2017 Ingo Schwarze <schwarze@openbsd.org>
5 * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 #include <sys/types.h>
36 #include "mandoc_aux.h"
41 #include "libmandoc.h"
43 #define REPARSE_LIMIT 1000
46 struct roff
*roff
; /* roff parser (!NULL) */
47 struct roff_man
*man
; /* man parser */
48 char *sodest
; /* filename pointed to by .so */
49 const char *file
; /* filename of current input file */
50 struct buf
*primary
; /* buffer currently being parsed */
51 struct buf
*secondary
; /* preprocessed copy of input */
52 const char *os_s
; /* default operating system */
53 mandocmsg mmsg
; /* warning/error message handler */
54 enum mandoclevel file_status
; /* status of current parse */
55 enum mandocerr mmin
; /* ignore messages below this */
56 int options
; /* parser options */
57 int gzip
; /* current input file is gzipped */
58 int filenc
; /* encoding of the current file */
59 int reparse_count
; /* finite interp. stack */
60 int line
; /* line number in the file */
63 static void choose_parser(struct mparse
*);
64 static void resize_buf(struct buf
*, size_t);
65 static int mparse_buf_r(struct mparse
*, struct buf
, size_t, int);
66 static int read_whole_file(struct mparse
*, const char *, int,
68 static void mparse_end(struct mparse
*);
69 static void mparse_parse_buffer(struct mparse
*, struct buf
,
72 static const enum mandocerr mandoclimits
[MANDOCLEVEL_MAX
] = {
82 static const char * const mandocerrs
[MANDOCERR_MAX
] = {
85 "base system convention",
89 "unknown architecture",
90 "operating system explicitly specified",
92 "referenced manual not found",
94 "generic style suggestion",
96 "legacy man(7) date format",
97 "lower case character in document title",
99 "typo in section name",
100 "unterminated quoted argument",
102 "consider using OS macro",
103 "errnos out of order",
105 "trailing delimiter",
106 "no blank before trailing delimiter",
107 "fill mode already enabled, skipping",
108 "fill mode already disabled, skipping",
109 "function name without markup",
110 "whitespace at end of input line",
115 /* related to the prologue */
116 "missing manual title, using UNTITLED",
117 "missing manual title, using \"\"",
118 "missing manual section, using \"\"",
119 "unknown manual section",
120 "missing date, using today's date",
121 "cannot parse date, using it verbatim",
122 "date in the future, using it anyway",
123 "missing Os macro, using \"\"",
124 "late prologue macro",
125 "prologue macros out of order",
127 /* related to document structure */
128 ".so is fragile, better use ln(1)",
130 "content before first section header",
131 "first section is not \"NAME\"",
132 "NAME section without Nm before Nd",
133 "NAME section without description",
134 "description not at the end of NAME",
135 "bad NAME section content",
136 "missing comma before name",
137 "missing description line, using \"\"",
138 "description line outside NAME section",
139 "sections out of conventional order",
140 "duplicate section title",
141 "unexpected section",
142 "cross reference to self",
144 "unusual Xr punctuation",
145 "AUTHORS section without An macro",
147 /* related to macros and nesting */
149 "macro neither callable nor escaped",
150 "skipping paragraph macro",
151 "moving paragraph macro out of list",
152 "skipping no-space macro",
153 "blocks badly nested",
154 "nested displays are not portable",
155 "moving content out of list",
156 "first macro on line",
158 "skipping blank line in line scope",
160 /* related to missing macro arguments */
161 "skipping empty request",
162 "conditional request controls empty scope",
163 "skipping empty macro",
165 "empty argument, using 0n",
166 "missing display type, using -ragged",
167 "list type is not the first argument",
168 "missing -width in -tag list, using 6n",
169 "missing utility name, using \"\"",
170 "missing function name, using \"\"",
171 "empty head in list item",
173 "missing argument, using next line",
174 "missing font type, using \\fR",
175 "unknown font type, using \\fR",
176 "nothing follows prefix",
177 "empty reference block",
178 "missing section argument",
179 "missing -std argument, adding it",
180 "missing option string, using \"\"",
181 "missing resource identifier, using \"\"",
182 "missing eqn box, using \"\"",
184 /* related to bad macro arguments */
185 "duplicate argument",
186 "skipping duplicate argument",
187 "skipping duplicate display type",
188 "skipping duplicate list type",
189 "skipping -width argument",
190 "wrong number of cells",
191 "unknown AT&T UNIX version",
192 "comma in function argument",
193 "parenthesis in function name",
194 "unknown library name",
195 "invalid content in Rs block",
196 "invalid Boolean argument",
197 "unknown font, skipping request",
198 "odd number of characters in request",
200 /* related to plain text */
201 "blank line in fill mode, using .sp",
202 "tab in filled text",
203 "new sentence, new line",
204 "invalid escape sequence",
205 "undefined string, using \"\"",
207 /* related to tables */
208 "tbl line starts with span",
209 "tbl column starts with span",
210 "skipping vertical bar in tbl layout",
214 /* related to tables */
215 "non-alphabetic character in tbl options",
216 "skipping unknown tbl option",
217 "missing tbl option argument",
218 "wrong tbl option argument size",
220 "invalid character in tbl layout",
221 "unmatched parenthesis in tbl layout",
222 "tbl without any data cells",
223 "ignoring data in spanned tbl cell",
224 "ignoring extra tbl data cells",
225 "data block open at end of tbl",
227 /* related to document structure and macros */
229 "duplicate prologue macro",
230 "skipping late title macro",
231 "input stack limit exceeded, infinite loop?",
232 "skipping bad character",
233 "skipping unknown macro",
234 "skipping insecure request",
235 "skipping item outside list",
236 "skipping column outside column list",
237 "skipping end of block that is not open",
238 "fewer RS blocks open, skipping",
239 "inserting missing end of block",
240 "appending missing end of block",
242 /* related to request and macro arguments */
243 "escaped character not allowed in a name",
244 "NOT IMPLEMENTED: Bd -file",
245 "skipping display without arguments",
246 "missing list type, using -item",
247 "argument is not numeric, using 1",
248 "missing manual name, using \"\"",
249 "uname(3) system call failed, using UNKNOWN",
250 "unknown standard specifier",
251 "skipping request without numeric argument",
252 "NOT IMPLEMENTED: .so with absolute path or \"..\"",
253 ".so request failed",
254 "skipping all arguments",
255 "skipping excess arguments",
258 "unsupported feature",
260 "unsupported control character",
261 "unsupported roff request",
262 "eqn delim option in tbl",
263 "unsupported tbl layout modifier",
264 "ignoring macro in table",
267 static const char * const mandoclevels
[MANDOCLEVEL_MAX
] = {
279 resize_buf(struct buf
*buf
, size_t initial
)
282 buf
->sz
= buf
->sz
> initial
/2 ? 2 * buf
->sz
: initial
;
283 buf
->buf
= mandoc_realloc(buf
->buf
, buf
->sz
);
287 choose_parser(struct mparse
*curp
)
293 * If neither command line arguments -mdoc or -man select
294 * a parser nor the roff parser found a .Dd or .TH macro
295 * yet, look ahead in the main input buffer.
298 if ((format
= roff_getformat(curp
->roff
)) == 0) {
299 cp
= curp
->primary
->buf
;
300 ep
= cp
+ curp
->primary
->sz
;
302 if (*cp
== '.' || *cp
== '\'') {
304 if (cp
[0] == 'D' && cp
[1] == 'd') {
305 format
= MPARSE_MDOC
;
308 if (cp
[0] == 'T' && cp
[1] == 'H') {
313 cp
= memchr(cp
, '\n', ep
- cp
);
320 if (format
== MPARSE_MDOC
) {
321 curp
->man
->macroset
= MACROSET_MDOC
;
322 if (curp
->man
->mdocmac
== NULL
)
323 curp
->man
->mdocmac
= roffhash_alloc(MDOC_Dd
, MDOC_MAX
);
325 curp
->man
->macroset
= MACROSET_MAN
;
326 if (curp
->man
->manmac
== NULL
)
327 curp
->man
->manmac
= roffhash_alloc(MAN_TH
, MAN_MAX
);
329 curp
->man
->first
->tok
= TOKEN_NONE
;
333 * Main parse routine for a buffer.
334 * It assumes encoding and line numbering are already set up.
335 * It can recurse directly (for invocations of user-defined
336 * macros, inline equations, and input line traps)
337 * and indirectly (for .so file inclusion).
340 mparse_buf_r(struct mparse
*curp
, struct buf blk
, size_t i
, int start
)
343 const char *save_file
;
345 size_t pos
; /* byte number in the ln buffer */
348 int lnn
; /* line number in the real file */
352 memset(&ln
, 0, sizeof(ln
));
358 if (0 == pos
&& '\0' == blk
.buf
[i
])
363 curp
->reparse_count
= 0;
366 curp
->filenc
& MPARSE_UTF8
&&
367 curp
->filenc
& MPARSE_LATIN1
)
368 curp
->filenc
= preconv_cue(&blk
, i
);
371 while (i
< blk
.sz
&& (start
|| blk
.buf
[i
] != '\0')) {
374 * When finding an unescaped newline character,
375 * leave the character loop to process the line.
376 * Skip a preceding carriage return, if any.
379 if ('\r' == blk
.buf
[i
] && i
+ 1 < blk
.sz
&&
380 '\n' == blk
.buf
[i
+ 1])
382 if ('\n' == blk
.buf
[i
]) {
389 * Make sure we have space for the worst
390 * case of 11 bytes: "\\[u10ffff]\0"
393 if (pos
+ 11 > ln
.sz
)
394 resize_buf(&ln
, 256);
397 * Encode 8-bit input.
402 if ( ! (curp
->filenc
&& preconv_encode(
403 &blk
, &i
, &ln
, &pos
, &curp
->filenc
))) {
404 mandoc_vmsg(MANDOCERR_CHAR_BAD
, curp
,
405 curp
->line
, pos
, "0x%x", c
);
413 * Exclude control characters.
416 if (c
== 0x7f || (c
< 0x20 && c
!= 0x09)) {
417 mandoc_vmsg(c
== 0x00 || c
== 0x04 ||
418 c
> 0x0a ? MANDOCERR_CHAR_BAD
:
419 MANDOCERR_CHAR_UNSUPP
,
420 curp
, curp
->line
, pos
, "0x%x", c
);
427 ln
.buf
[pos
++] = blk
.buf
[i
++];
430 if (pos
+ 1 >= ln
.sz
)
431 resize_buf(&ln
, 256);
433 if (i
== blk
.sz
|| blk
.buf
[i
] == '\0')
434 ln
.buf
[pos
++] = '\n';
438 * A significant amount of complexity is contained by
439 * the roff preprocessor. It's line-oriented but can be
440 * expressed on one line, so we need at times to
441 * readjust our starting point and re-run it. The roff
442 * preprocessor can also readjust the buffers with new
443 * data, so we pass them in wholesale.
449 * Maintain a lookaside buffer of all parsed lines. We
450 * only do this if mparse_keep() has been invoked (the
451 * buffer may be accessed with mparse_getkeep()).
454 if (curp
->secondary
) {
455 curp
->secondary
->buf
= mandoc_realloc(
456 curp
->secondary
->buf
,
457 curp
->secondary
->sz
+ pos
+ 2);
458 memcpy(curp
->secondary
->buf
+
461 curp
->secondary
->sz
+= pos
;
463 [curp
->secondary
->sz
] = '\n';
464 curp
->secondary
->sz
++;
466 [curp
->secondary
->sz
] = '\0';
469 rr
= roff_parseln(curp
->roff
, curp
->line
, &ln
, &of
);
473 if (++curp
->reparse_count
> REPARSE_LIMIT
)
474 mandoc_msg(MANDOCERR_ROFFLOOP
, curp
,
475 curp
->line
, pos
, NULL
);
476 else if (mparse_buf_r(curp
, ln
, of
, 0) == 1 ||
484 pos
= strlen(ln
.buf
);
492 if ( ! (curp
->options
& MPARSE_SO
) &&
493 (i
>= blk
.sz
|| blk
.buf
[i
] == '\0')) {
494 curp
->sodest
= mandoc_strdup(ln
.buf
+ of
);
499 * We remove `so' clauses from our lookaside
500 * buffer because we're going to descend into
501 * the file recursively.
504 curp
->secondary
->sz
-= pos
+ 1;
505 save_file
= curp
->file
;
506 if ((fd
= mparse_open(curp
, ln
.buf
+ of
)) != -1) {
507 mparse_readfd(curp
, fd
, ln
.buf
+ of
);
509 curp
->file
= save_file
;
511 curp
->file
= save_file
;
512 mandoc_vmsg(MANDOCERR_SO_FAIL
,
513 curp
, curp
->line
, pos
,
514 ".so %s", ln
.buf
+ of
);
515 ln
.sz
= mandoc_asprintf(&cp
,
516 ".sp\nSee the file %s.\n.sp",
521 mparse_buf_r(curp
, ln
, of
, 0);
529 if (curp
->man
->macroset
== MACROSET_NONE
)
532 if ((curp
->man
->macroset
== MACROSET_MDOC
?
533 mdoc_parseln(curp
->man
, curp
->line
, ln
.buf
, of
) :
534 man_parseln(curp
->man
, curp
->line
, ln
.buf
, of
)) == 2)
537 /* Temporary buffers typically are not full. */
539 if (0 == start
&& '\0' == blk
.buf
[i
])
542 /* Start the next input line. */
552 read_whole_file(struct mparse
*curp
, const char *file
, int fd
,
553 struct buf
*fb
, int *with_mmap
)
560 if (fstat(fd
, &st
) == -1) {
561 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
562 "fstat: %s", strerror(errno
));
567 * If we're a regular file, try just reading in the whole entry
568 * via mmap(). This is faster than reading it into blocks, and
569 * since each file is only a few bytes to begin with, I'm not
570 * concerned that this is going to tank any machines.
573 if (curp
->gzip
== 0 && S_ISREG(st
.st_mode
)) {
574 if (st
.st_size
> 0x7fffffff) {
575 mandoc_msg(MANDOCERR_TOOLARGE
, curp
, 0, 0, NULL
);
579 fb
->sz
= (size_t)st
.st_size
;
580 fb
->buf
= mmap(NULL
, fb
->sz
, PROT_READ
, MAP_SHARED
, fd
, 0);
581 if (fb
->buf
!= MAP_FAILED
)
586 if ((gz
= gzdopen(fd
, "rb")) == NULL
) {
587 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
588 "gzdopen: %s", strerror(errno
));
595 * If this isn't a regular file (like, say, stdin), then we must
596 * go the old way and just read things in bit by bit.
605 if (fb
->sz
== (1U << 31)) {
606 mandoc_msg(MANDOCERR_TOOLARGE
, curp
,
610 resize_buf(fb
, 65536);
613 gzread(gz
, fb
->buf
+ (int)off
, fb
->sz
- off
) :
614 read(fd
, fb
->buf
+ (int)off
, fb
->sz
- off
);
620 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
621 "read: %s", strerror(errno
));
633 mparse_end(struct mparse
*curp
)
635 if (curp
->man
->macroset
== MACROSET_NONE
)
636 curp
->man
->macroset
= MACROSET_MAN
;
637 if (curp
->man
->macroset
== MACROSET_MDOC
)
638 mdoc_endparse(curp
->man
);
640 man_endparse(curp
->man
);
641 roff_endparse(curp
->roff
);
645 mparse_parse_buffer(struct mparse
*curp
, struct buf blk
, const char *file
)
647 struct buf
*svprimary
;
650 static int recursion_depth
;
652 if (64 < recursion_depth
) {
653 mandoc_msg(MANDOCERR_ROFFLOOP
, curp
, curp
->line
, 0, NULL
);
657 /* Line number is per-file. */
660 svprimary
= curp
->primary
;
661 curp
->primary
= &blk
;
665 /* Skip an UTF-8 byte order mark. */
666 if (curp
->filenc
& MPARSE_UTF8
&& blk
.sz
> 2 &&
667 (unsigned char)blk
.buf
[0] == 0xef &&
668 (unsigned char)blk
.buf
[1] == 0xbb &&
669 (unsigned char)blk
.buf
[2] == 0xbf) {
671 curp
->filenc
&= ~MPARSE_LATIN1
;
675 mparse_buf_r(curp
, blk
, offset
, 1);
677 if (--recursion_depth
== 0)
680 curp
->primary
= svprimary
;
685 mparse_readmem(struct mparse
*curp
, void *buf
, size_t len
,
693 mparse_parse_buffer(curp
, blk
, file
);
694 return curp
->file_status
;
698 * Read the whole file into memory and call the parsers.
699 * Called recursively when an .so request is encountered.
702 mparse_readfd(struct mparse
*curp
, int fd
, const char *file
)
708 if (read_whole_file(curp
, file
, fd
, &blk
, &with_mmap
)) {
709 save_filenc
= curp
->filenc
;
710 curp
->filenc
= curp
->options
&
711 (MPARSE_UTF8
| MPARSE_LATIN1
);
712 mparse_parse_buffer(curp
, blk
, file
);
713 curp
->filenc
= save_filenc
;
715 munmap(blk
.buf
, blk
.sz
);
719 return curp
->file_status
;
723 mparse_open(struct mparse
*curp
, const char *file
)
729 cp
= strrchr(file
, '.');
730 curp
->gzip
= (cp
!= NULL
&& ! strcmp(cp
+ 1, "gz"));
732 /* First try to use the filename as it is. */
734 if ((fd
= open(file
, O_RDONLY
)) != -1)
738 * If that doesn't work and the filename doesn't
739 * already end in .gz, try appending .gz.
743 mandoc_asprintf(&cp
, "%s.gz", file
);
744 fd
= open(cp
, O_RDONLY
);
752 /* Neither worked, give up. */
754 mandoc_msg(MANDOCERR_FILE
, curp
, 0, 0, strerror(errno
));
759 mparse_alloc(int options
, enum mandocerr mmin
, mandocmsg mmsg
,
760 enum mandoc_os os_e
, const char *os_s
)
764 curp
= mandoc_calloc(1, sizeof(struct mparse
));
766 curp
->options
= options
;
771 curp
->roff
= roff_alloc(curp
, options
);
772 curp
->man
= roff_man_alloc(curp
->roff
, curp
, curp
->os_s
,
773 curp
->options
& MPARSE_QUICK
? 1 : 0);
774 if (curp
->options
& MPARSE_MDOC
) {
775 curp
->man
->macroset
= MACROSET_MDOC
;
776 if (curp
->man
->mdocmac
== NULL
)
777 curp
->man
->mdocmac
= roffhash_alloc(MDOC_Dd
, MDOC_MAX
);
778 } else if (curp
->options
& MPARSE_MAN
) {
779 curp
->man
->macroset
= MACROSET_MAN
;
780 if (curp
->man
->manmac
== NULL
)
781 curp
->man
->manmac
= roffhash_alloc(MAN_TH
, MAN_MAX
);
783 curp
->man
->first
->tok
= TOKEN_NONE
;
784 curp
->man
->meta
.os_e
= os_e
;
789 mparse_reset(struct mparse
*curp
)
791 roff_reset(curp
->roff
);
792 roff_man_reset(curp
->man
);
798 curp
->secondary
->sz
= 0;
800 curp
->file_status
= MANDOCLEVEL_OK
;
805 mparse_free(struct mparse
*curp
)
808 roffhash_free(curp
->man
->mdocmac
);
809 roffhash_free(curp
->man
->manmac
);
810 roff_man_free(curp
->man
);
811 roff_free(curp
->roff
);
813 free(curp
->secondary
->buf
);
815 free(curp
->secondary
);
821 mparse_result(struct mparse
*curp
, struct roff_man
**man
,
825 if (sodest
&& NULL
!= (*sodest
= curp
->sodest
)) {
834 mparse_updaterc(struct mparse
*curp
, enum mandoclevel
*rc
)
836 if (curp
->file_status
> *rc
)
837 *rc
= curp
->file_status
;
841 mandoc_vmsg(enum mandocerr t
, struct mparse
*m
,
842 int ln
, int pos
, const char *fmt
, ...)
848 (void)vsnprintf(buf
, sizeof(buf
), fmt
, ap
);
851 mandoc_msg(t
, m
, ln
, pos
, buf
);
855 mandoc_msg(enum mandocerr er
, struct mparse
*m
,
856 int ln
, int col
, const char *msg
)
858 enum mandoclevel level
;
860 if (er
< m
->mmin
&& er
!= MANDOCERR_FILE
)
863 level
= MANDOCLEVEL_UNSUPP
;
864 while (er
< mandoclimits
[level
])
868 (*m
->mmsg
)(er
, level
, m
->file
, ln
, col
, msg
);
870 if (m
->file_status
< level
)
871 m
->file_status
= level
;
875 mparse_strerror(enum mandocerr er
)
878 return mandocerrs
[er
];
882 mparse_strlevel(enum mandoclevel lvl
)
884 return mandoclevels
[lvl
];
888 mparse_keep(struct mparse
*p
)
891 assert(NULL
== p
->secondary
);
892 p
->secondary
= mandoc_calloc(1, sizeof(struct buf
));
896 mparse_getkeep(const struct mparse
*p
)
899 assert(p
->secondary
);
900 return p
->secondary
->sz
? p
->secondary
->buf
: NULL
;