src/encoding/encoding.c

   1 /* Stream reading and decoding (mostly decompression) */
   2
   3 #ifdef HAVE_CONFIG_H
   4 #include "config.h"
   5 #endif
   6
   7 #include <errno.h>
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <sys/stat.h> /* OS/2 needs this after sys/types.h */
  11 #include <sys/types.h>
  12 #ifdef HAVE_FCNTL_H
  13 #include <fcntl.h> /* OS/2 needs this after sys/types.h */
  14 #endif
  15 #ifdef HAVE_UNISTD_H
  16 #include <unistd.h>
  17 #endif
  18
  19 #include "elinks.h"
  20
  21 #include "config/options.h"
  22 #include "encoding/encoding.h"
  23 #include "network/state.h"
  24 #include "osdep/osdep.h"
  25 #include "util/memory.h"
  26 #include "util/string.h"
  27
  28
  29 /*************************************************************************
  30   Dummy encoding (ENCODING_NONE)
  31 *************************************************************************/
  32
  33 struct dummy_enc_data {
  34         int fd;
  35 };
  36
  37 static int
  38 dummy_open(struct stream_encoded *stream, int fd)
  39 {
  40         stream->data = mem_alloc(sizeof(struct dummy_enc_data));
  41         if (!stream->data) return -1;
  42
  43         ((struct dummy_enc_data *) stream->data)->fd = fd;
  44
  45         return 0;
  46 }
  47
  48 static int
  49 dummy_read(struct stream_encoded *stream, unsigned char *data, int len)
  50 {
  51         return safe_read(((struct dummy_enc_data *) stream->data)->fd, data, len);
  52 }
  53
  54 static unsigned char *
  55 dummy_decode_buffer(struct stream_encoded *stream, unsigned char *data, int len, int *new_len)
  56 {
  57         unsigned char *buffer = memacpy(data, len);
  58
  59         if (!buffer) return NULL;
  60
  61         *new_len = len;
  62         return buffer;
  63 }
  64
  65 static void
  66 dummy_close(struct stream_encoded *stream)
  67 {
  68         close(((struct dummy_enc_data *) stream->data)->fd);
  69         mem_free(stream->data);
  70 }
  71
  72 static const unsigned char *const dummy_extensions[] = { NULL };
  73
  74 static const struct decoding_backend dummy_decoding_backend = {
  75         "none",
  76         dummy_extensions,
  77         dummy_open,
  78         dummy_read,
  79         dummy_decode_buffer,
  80         dummy_close,
  81 };
  82
  83
  84 /* Dynamic backend area */
  85
  86 #include "encoding/bzip2.h"
  87 #include "encoding/deflate.h"
  88 #include "encoding/lzma.h"
  89
  90 static const struct decoding_backend *const decoding_backends[] = {
  91         &dummy_decoding_backend,
  92         &gzip_decoding_backend,
  93         &bzip2_decoding_backend,
  94         &lzma_decoding_backend,
  95         &deflate_decoding_backend,
  96 };
  97
  98
  99 /*************************************************************************
 100   Public functions
 101 *************************************************************************/
 102
 103
 104 /* Associates encoded stream with a fd. */
 105 struct stream_encoded *
 106 open_encoded(int fd, enum stream_encoding encoding)
 107 {
 108         struct stream_encoded *stream;
 109
 110         stream = mem_alloc(sizeof(*stream));
 111         if (!stream) return NULL;
 112
 113         stream->encoding = encoding;
 114         if (decoding_backends[stream->encoding]->open(stream, fd) >= 0)
 115                 return stream;
 116
 117         mem_free(stream);
 118         return NULL;
 119 }
 120
 121 /* Read available data from stream and decode them. Note that when data change
 122  * their size during decoding, 'len' indicates desired size of _returned_ data,
 123  * not desired size of data read from stream. */
 124 int
 125 read_encoded(struct stream_encoded *stream, unsigned char *data, int len)
 126 {
 127         return decoding_backends[stream->encoding]->read(stream, data, len);
 128 }
 129
 130 /* Decode an entire file from a buffer. This function is not suitable
 131  * for parts of files. @data contains the original data, @len bytes
 132  * long. The resulting decoded data chunk is *@new_len bytes long. */
 133 unsigned char *
 134 decode_encoded_buffer(struct stream_encoded *stream, enum stream_encoding encoding, unsigned char *data, int len,
 135                       int *new_len)
 136 {
 137         return decoding_backends[encoding]->decode_buffer(stream, data, len, new_len);
 138 }
 139
 140 /* Closes encoded stream. Note that fd associated with the stream will be
 141  * closed here. */
 142 void
 143 close_encoded(struct stream_encoded *stream)
 144 {
 145         decoding_backends[stream->encoding]->close(stream);
 146         mem_free(stream);
 147 }
 148
 149
 150 /* Return a list of extensions associated with that encoding. */
 151 const unsigned char *const *listext_encoded(enum stream_encoding encoding)
 152 {
 153         return decoding_backends[encoding]->extensions;
 154 }
 155
 156 enum stream_encoding
 157 guess_encoding(unsigned char *filename)
 158 {
 159         int fname_len = strlen(filename);
 160         unsigned char *fname_end = filename + fname_len;
 161         int enc;
 162
 163         for (enc = 1; enc < ENCODINGS_KNOWN; enc++) {
 164                 const unsigned char *const *ext = decoding_backends[enc]->extensions;
 165
 166                 while (ext && *ext) {
 167                         int len = strlen(*ext);
 168
 169                         if (fname_len >= len && !strcmp(fname_end - len, *ext))
 170                                 return enc;
 171
 172                         ext++;
 173                 }
 174         }
 175
 176         return ENCODING_NONE;
 177 }
 178
 179 const unsigned char *
 180 get_encoding_name(enum stream_encoding encoding)
 181 {
 182         return decoding_backends[encoding]->name;
 183 }
 184
 185
 186 /* File reading */
 187
 188 /* Tries to open @prefixname with each of the supported encoding extensions
 189  * appended. */
 190 static inline enum stream_encoding
 191 try_encoding_extensions(struct string *filename, int *fd)
 192 {
 193         int length = filename->length;
 194         int encoding;
 195
 196         /* No file of that name was found, try some others names. */
 197         for (encoding = 1; encoding < ENCODINGS_KNOWN; encoding++) {
 198                 const unsigned char *const *ext = listext_encoded(encoding);
 199
 200                 for (; ext && *ext; ext++) {
 201                         add_to_string(filename, *ext);
 202
 203                         /* We try with some extensions. */
 204                         *fd = open(filename->source, O_RDONLY | O_NOCTTY);
 205
 206                         if (*fd >= 0)
 207                                 /* Ok, found one, use it. */
 208                                 return encoding;
 209
 210                         filename->source[length] = 0;
 211                         filename->length = length;
 212                 }
 213         }
 214
 215         return ENCODING_NONE;
 216 }
 217
 218 /** Reads the file from @a stream in chunks of size @a readsize.
 219  *
 220  * @a stream should be in blocking mode.  If it is in non-blocking
 221  * mode, this function can return an empty string in @a page just
 222  * because no more data is available yet, and the caller cannot know
 223  * whether the true end of the stream has been reached.
 224  *
 225  * @return a connection state. S_OK if all is well. */
 226 struct connection_state
 227 read_file(struct stream_encoded *stream, int readsize, struct string *page)
 228 {
 229         if (!init_string(page)) return connection_state(S_OUT_OF_MEM);
 230
 231         /* We read with granularity of stt.st_size (given as @readsize) - this
 232          * does best job for uncompressed files, and doesn't hurt for
 233          * compressed ones anyway - very large files usually tend to inflate
 234          * fast anyway. At least I hope ;).  --pasky */
 235         /* Also there because of bug in Linux. Read returns -EACCES when
 236          * reading 0 bytes to invalid address so ensure never to try and
 237          * allocate zero number of bytes. */
 238         if (!readsize) readsize = 4096;
 239
 240         while (realloc_string(page, page->length + readsize)) {
 241                 unsigned char *string_pos = page->source + page->length;
 242                 int readlen = read_encoded(stream, string_pos, readsize);
 243
 244                 if (readlen < 0) {
 245                         done_string(page);
 246
 247                         /* If it is some I/O error (and errno is set) that will
 248                          * do. Since errno == 0 == S_WAIT and we cannot have
 249                          * that. */
 250                         if (errno)
 251                                 return connection_state_for_errno(errno);
 252
 253                         /* FIXME: This is indeed an internal error. If readed from a
 254                          * corrupted encoded file nothing or only some of the
 255                          * data will be read. */
 256                         return connection_state(S_ENCODE_ERROR);
 257
 258                 } else if (readlen == 0) {
 259                         /* NUL-terminate just in case */
 260                         page->source[page->length] = '\0';
 261                         return connection_state(S_OK);
 262                 }
 263
 264                 page->length += readlen;
 265 #if 0
 266                 /* This didn't work so well as it should (I had to implement
 267                  * end of stream handling to bzip2 anyway), so I rather
 268                  * disabled this. */
 269                 if (readlen < readsize) {
 270                         /* This is much safer. It should always mean that we
 271                          * already read everything possible, and it permits us
 272                          * more elegant of handling end of file with bzip2. */
 273                         break;
 274                 }
 275 #endif
 276         }
 277
 278         done_string(page);
 279         return connection_state(S_OUT_OF_MEM);
 280 }
 281
 282 static inline int
 283 is_stdin_pipe(struct stat *stt, struct string *filename)
 284 {
 285         /* On Mac OS X, /dev/stdin has type S_IFSOCK. (bug 616) */
 286         return !strlcmp(filename->source, filename->length, "/dev/stdin", 10)
 287                 && (
 288 #ifdef S_ISSOCK
 289                         S_ISSOCK(stt->st_mode) ||
 290 #endif
 291                         S_ISFIFO(stt->st_mode));
 292 }
 293
 294 struct connection_state
 295 read_encoded_file(struct string *filename, struct string *page)
 296 {
 297         struct stream_encoded *stream;
 298         struct stat stt;
 299         enum stream_encoding encoding = ENCODING_NONE;
 300         int fd = open(filename->source, O_RDONLY | O_NOCTTY);
 301         struct connection_state state = connection_state_for_errno(errno);
 302
 303         if (fd == -1 && get_opt_bool("protocol.file.try_encoding_extensions", NULL)) {
 304                 encoding = try_encoding_extensions(filename, &fd);
 305
 306         } else if (fd != -1) {
 307                 encoding = guess_encoding(filename->source);
 308         }
 309
 310         if (fd == -1) {
 311 #ifdef HAVE_SYS_CYGWIN_H
 312                 /* There is no /dev/stdin on Cygwin. */
 313                 if (!strlcmp(filename->source, filename->length, "/dev/stdin", 10)) {
 314                         fd = STDIN_FILENO;
 315                 } else
 316 #endif
 317                 return state;
 318         }
 319
 320         /* Some file was opened so let's get down to bi'ness */
 321         set_bin(fd);
 322
 323         /* Do all the necessary checks before trying to read the file.
 324          * @state code is used to block further progress. */
 325         if (fstat(fd, &stt)) {
 326                 state = connection_state_for_errno(errno);
 327
 328         } else if (!S_ISREG(stt.st_mode) && encoding != ENCODING_NONE) {
 329                 /* We only want to open regular encoded files. */
 330                 /* Leave @state being the saved errno */
 331
 332         } else if (!S_ISREG(stt.st_mode) && !is_stdin_pipe(&stt, filename)
 333                    && !get_opt_bool("protocol.file.allow_special_files", NULL)) {
 334                 state = connection_state(S_FILE_TYPE);
 335
 336         } else if (!(stream = open_encoded(fd, encoding))) {
 337                 state = connection_state(S_OUT_OF_MEM);
 338
 339         } else {
 340                 int readsize = (int) stt.st_size;
 341
 342                 /* Check if st_size will cause overflow. */
 343                 /* FIXME: See bug 497 for info about support for big files. */
 344                 if (readsize != stt.st_size || readsize < 0) {
 345 #ifdef EFBIG
 346                         state = connection_state_for_errno(EFBIG);
 347 #else
 348                         state = connection_state(S_FILE_ERROR);
 349 #endif
 350
 351                 } else {
 352                         state = read_file(stream, stt.st_size, page);
 353                 }
 354                 close_encoded(stream);
 355         }
 356
 357         close(fd);
 358         return state;
 359 }