1 /* Id: preconv.c,v 1.6 2013/06/02 03:52:21 schwarze Exp */
3 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
34 * The read_whole_file() and resize_buf() functions are copied from
35 * read.c, including all dependency code.
39 ENC_UTF_8
, /* UTF-8 */
40 ENC_US_ASCII
, /* US-ASCII */
41 ENC_LATIN_1
, /* Latin-1 */
46 char *buf
; /* binary input buffer */
47 size_t sz
; /* size of binary buffer */
48 size_t offs
; /* starting buffer offset */
53 int (*conv
)(const struct buf
*);
56 static int cue_enc(const struct buf
*, size_t *, enum enc
*);
57 static int conv_latin_1(const struct buf
*);
58 static int conv_us_ascii(const struct buf
*);
59 static int conv_utf_8(const struct buf
*);
60 static int read_whole_file(const char *, int,
62 static void resize_buf(struct buf
*, size_t);
63 static void usage(void);
65 static const struct encode encs
[ENC__MAX
] = {
66 { "utf-8", conv_utf_8
}, /* ENC_UTF_8 */
67 { "us-ascii", conv_us_ascii
}, /* ENC_US_ASCII */
68 { "latin-1", conv_latin_1
}, /* ENC_LATIN_1 */
71 static const char *progname
;
77 fprintf(stderr
, "usage: %s "
80 "[file]\n", progname
);
84 conv_latin_1(const struct buf
*b
)
90 cp
= b
->buf
+ (int)b
->offs
;
93 * Latin-1 falls into the first 256 code-points of Unicode, so
94 * there's no need for any sort of translation. Just make the
95 * 8-bit characters use the Unicode escape.
96 * Note that binary values 128 < v < 160 are passed through
97 * unmodified to mandoc.
100 for (i
= b
->offs
; i
< b
->sz
; i
++) {
101 cu
= (unsigned char)*cp
++;
102 cu
< 160U ? putchar(cu
) : printf("\\[u%.4X]", cu
);
109 conv_us_ascii(const struct buf
*b
)
113 * US-ASCII has no conversion since it falls into the first 128
117 fwrite(b
->buf
, 1, b
->sz
, stdout
);
122 conv_utf_8(const struct buf
*b
)
131 cp
= b
->buf
+ (int)b
->offs
;
136 /* Quick test for big-endian value. */
138 if ( ! (*((const char *)(&one
))))
141 for (i
= b
->offs
; i
< b
->sz
; i
++) {
142 cu
= (unsigned char)*cp
++;
144 if ( ! (cu
& 128) || (cu
& 64)) {
145 /* Bad sequence header. */
149 /* Accept only legitimate bit patterns. */
151 if (cu
> 191 || cu
< 128) {
152 /* Bad in-sequence bits. */
156 accum
|= (cu
& 63) << --state
* 6;
159 * Accum is held in little-endian order as
160 * stipulated by the UTF-8 sequence coding. We
161 * need to convert to a native big-endian if our
162 * architecture requires it.
165 if (0 == state
&& be
)
166 accum
= (accum
>> 24) |
167 ((accum
<< 8) & 0x00FF0000) |
168 ((accum
>> 8) & 0x0000FF00) |
172 accum
< 128U ? putchar(accum
) :
173 printf("\\[u%.4X]", accum
);
176 } else if (cu
& (1 << 7)) {
178 * Entering a UTF-8 state: if we encounter a
179 * UTF-8 bitmask, calculate the expected UTF-8
182 for (state
= 0; state
< 7; state
++)
183 if ( ! (cu
& (1 << (7 - state
))))
186 /* Accept only legitimate bit patterns. */
190 if (cu
<= 244 && cu
>= 240) {
191 accum
= (cu
& 7) << 18;
194 /* Bad 4-sequence start bits. */
197 if (cu
<= 239 && cu
>= 224) {
198 accum
= (cu
& 15) << 12;
201 /* Bad 3-sequence start bits. */
204 if (cu
<= 223 && cu
>= 194) {
205 accum
= (cu
& 31) << 6;
208 /* Bad 2-sequence start bits. */
211 /* Bad sequence bit mask. */
220 /* Bad trailing bits. */
228 resize_buf(struct buf
*buf
, size_t initial
)
231 buf
->sz
= buf
->sz
> initial
/ 2 ?
232 2 * buf
->sz
: initial
;
234 buf
->buf
= realloc(buf
->buf
, buf
->sz
);
235 if (NULL
== buf
->buf
) {
242 read_whole_file(const char *f
, int fd
,
243 struct buf
*fb
, int *with_mmap
)
250 if (-1 == fstat(fd
, &st
)) {
256 * If we're a regular file, try just reading in the whole entry
257 * via mmap(). This is faster than reading it into blocks, and
258 * since each file is only a few bytes to begin with, I'm not
259 * concerned that this is going to tank any machines.
262 if (S_ISREG(st
.st_mode
) && st
.st_size
>= (1U << 31)) {
263 fprintf(stderr
, "%s: input too large\n", f
);
267 if (S_ISREG(st
.st_mode
)) {
269 fb
->sz
= (size_t)st
.st_size
;
270 fb
->buf
= mmap(NULL
, fb
->sz
, PROT_READ
, MAP_SHARED
, fd
, 0);
271 if (fb
->buf
!= MAP_FAILED
)
277 * If this isn't a regular file (like, say, stdin), then we must
278 * go the old way and just read things in bit by bit.
286 if (off
== fb
->sz
&& fb
->sz
== (1U << 31)) {
287 fprintf(stderr
, "%s: input too large\n", f
);
292 resize_buf(fb
, 65536);
294 ssz
= read(fd
, fb
->buf
+ (int)off
, fb
->sz
- off
);
312 cue_enc(const struct buf
*b
, size_t *offs
, enum enc
*enc
)
314 const char *ln
, *eoln
, *eoph
;
315 size_t sz
, phsz
, nsz
;
318 ln
= b
->buf
+ (int)*offs
;
321 /* Look for the end-of-line. */
323 if (NULL
== (eoln
= memchr(ln
, '\n', sz
)))
326 /* Set next-line marker. */
328 *offs
= (size_t)((eoln
+ 1) - b
->buf
);
330 /* Check if we have the correct header/trailer. */
332 if ((sz
= (size_t)(eoln
- ln
)) < 10 ||
333 memcmp(ln
, ".\\\" -*-", 7) ||
334 memcmp(eoln
- 3, "-*-", 3))
337 /* Move after the header and adjust for the trailer. */
343 while (sz
> 0 && ' ' == *ln
) {
350 /* Find the end-of-phrase marker (or eoln). */
352 if (NULL
== (eoph
= memchr(ln
, ';', sz
)))
357 /* Only account for the "coding" phrase. */
359 if ((phsz
= (size_t)(eoph
- ln
)) < 7 ||
360 strncasecmp(ln
, "coding:", 7)) {
369 while (sz
> 0 && ' ' == *ln
) {
376 /* Check us against known encodings. */
378 for (i
= 0; i
< (int)ENC__MAX
; i
++) {
379 nsz
= strlen(encs
[i
].name
);
382 if (strncasecmp(ln
, encs
[i
].name
, nsz
))
389 /* Unknown encoding. */
399 main(int argc
, char *argv
[])
401 int i
, ch
, map
, fd
, rc
;
405 unsigned char bom
[3] = { 0xEF, 0xBB, 0xBF };
410 progname
= strrchr(argv
[0], '/');
411 if (progname
== NULL
)
419 enc
= def
= ENC__MAX
;
422 memset(&b
, 0, sizeof(struct buf
));
424 while (-1 != (ch
= getopt(argc
, argv
, "D:e:rdvh")))
429 for (i
= 0; i
< (int)ENC__MAX
; i
++) {
430 if (strcasecmp(optarg
, encs
[i
].name
))
434 if (i
< (int)ENC__MAX
) {
442 fprintf(stderr
, "%s: Bad encoding\n", optarg
);
443 return(EXIT_FAILURE
);
449 /* Compatibility with GNU preconv. */
452 /* Compatibility with GNU preconv. */
456 return(EXIT_FAILURE
);
463 * Open and read the first argument on the command-line.
464 * If we don't have one, we default to stdin.
469 fd
= open(fn
, O_RDONLY
, 0);
472 return(EXIT_FAILURE
);
476 if ( ! read_whole_file(fn
, fd
, &b
, &map
))
479 /* Try to read the UTF-8 BOM. */
482 if (b
.sz
> 3 && 0 == memcmp(b
.buf
, bom
, 3)) {
487 /* Try reading from the "-*-" cue. */
489 if (ENC__MAX
== enc
) {
491 ch
= cue_enc(&b
, &offs
, &enc
);
493 ch
= cue_enc(&b
, &offs
, &enc
);
497 * No encoding has been detected.
498 * Thus, we either fall into our default encoder, if specified,
499 * or use Latin-1 if all else fails.
503 enc
= ENC__MAX
== def
? ENC_LATIN_1
: def
;
505 if ( ! (*encs
[(int)enc
].conv
)(&b
)) {
506 fprintf(stderr
, "%s: Bad encoding\n", fn
);
519 if (fd
> STDIN_FILENO
)