src/lib/compress/compress_lzma.c

   1 /* Copyright (c) 2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2021, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file compress_lzma.c
   8  * \brief Compression backend for LZMA.
   9  *
  10  * This module should never be invoked directly. Use the compress module
  11  * instead.
  12  **/
  13
  14 #include "orconfig.h"
  15
  16 #include "lib/compress/compress.h"
  17 #include "lib/compress/compress_lzma.h"
  18 #include "lib/log/log.h"
  19 #include "lib/log/util_bug.h"
  20 #include "lib/malloc/malloc.h"
  21 #include "lib/thread/threads.h"
  22
  23 #ifdef HAVE_LZMA
  24 #include <lzma.h>
  25 #endif
  26
  27 /** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */
  28 #define MEMORY_LIMIT (16 * 1024 * 1024)
  29
  30 /** Total number of bytes allocated for LZMA state. */
  31 static atomic_counter_t total_lzma_allocation;
  32
  33 #ifdef HAVE_LZMA
  34 /** Given <b>level</b> return the memory level. */
  35 static int
  36 memory_level(compression_level_t level)
  37 {
  38   switch (level) {
  39     default:
  40     case BEST_COMPRESSION:
  41     case HIGH_COMPRESSION: return 6;
  42     case MEDIUM_COMPRESSION: return 4;
  43     case LOW_COMPRESSION: return 2;
  44   }
  45 }
  46
  47 /** Convert a given <b>error</b> to a human readable error string. */
  48 static const char *
  49 lzma_error_str(lzma_ret error)
  50 {
  51   switch (error) {
  52     case LZMA_OK:
  53       return "Operation completed successfully";
  54     case LZMA_STREAM_END:
  55       return "End of stream";
  56     case LZMA_NO_CHECK:
  57       return "Input stream lacks integrity check";
  58     case LZMA_UNSUPPORTED_CHECK:
  59       return "Unable to calculate integrity check";
  60     case LZMA_GET_CHECK:
  61       return "Integrity check available";
  62     case LZMA_MEM_ERROR:
  63       return "Unable to allocate memory";
  64     case LZMA_MEMLIMIT_ERROR:
  65       return "Memory limit reached";
  66     case LZMA_FORMAT_ERROR:
  67       return "Unknown file format";
  68     case LZMA_OPTIONS_ERROR:
  69       return "Unsupported options";
  70     case LZMA_DATA_ERROR:
  71       return "Corrupt input data";
  72     case LZMA_BUF_ERROR:
  73       return "Unable to progress";
  74     case LZMA_PROG_ERROR:
  75       return "Programming error";
  76 #if LZMA_VERSION >= 50030010
  77     case LZMA_SEEK_NEEDED:
  78       // This can be returned by the .xz file_info decoder but with
  79       // lzma_alone_decoder/encoder as we use, it should never be seen.
  80       return "Seek needed";
  81 #endif
  82 #if LZMA_VERSION >= 50030020
  83     case LZMA_RET_INTERNAL1:
  84     case LZMA_RET_INTERNAL2:
  85     case LZMA_RET_INTERNAL3:
  86     case LZMA_RET_INTERNAL4:
  87     case LZMA_RET_INTERNAL5:
  88     case LZMA_RET_INTERNAL6:
  89     case LZMA_RET_INTERNAL7:
  90     case LZMA_RET_INTERNAL8:
  91       FALLTHROUGH;
  92 #endif
  93     default:
  94       return "Unknown LZMA error";
  95   }
  96 }
  97 #endif /* defined(HAVE_LZMA) */
  98
  99 /** Return 1 if LZMA compression is supported; otherwise 0. */
 100 int
 101 tor_lzma_method_supported(void)
 102 {
 103 #ifdef HAVE_LZMA
 104   return 1;
 105 #else
 106   return 0;
 107 #endif
 108 }
 109
 110 /** Return a string representation of the version of the currently running
 111  * version of liblzma. Returns NULL if LZMA is unsupported. */
 112 const char *
 113 tor_lzma_get_version_str(void)
 114 {
 115 #ifdef HAVE_LZMA
 116   return lzma_version_string();
 117 #else
 118   return NULL;
 119 #endif
 120 }
 121
 122 /** Return a string representation of the version of liblzma used at
 123  * compilation time. Returns NULL if LZMA is unsupported. */
 124 const char *
 125 tor_lzma_get_header_version_str(void)
 126 {
 127 #ifdef HAVE_LZMA
 128   return LZMA_VERSION_STRING;
 129 #else
 130   return NULL;
 131 #endif
 132 }
 133
 134 /** Internal LZMA state for incremental compression/decompression.
 135  * The body of this struct is not exposed. */
 136 struct tor_lzma_compress_state_t {
 137 #ifdef HAVE_LZMA
 138   lzma_stream stream; /**< The LZMA stream. */
 139 #endif
 140
 141   int compress; /**< True if we are compressing; false if we are inflating */
 142
 143   /** Number of bytes read so far.  Used to detect compression bombs. */
 144   size_t input_so_far;
 145   /** Number of bytes written so far.  Used to detect compression bombs. */
 146   size_t output_so_far;
 147
 148   /** Approximate number of bytes allocated for this object. */
 149   size_t allocation;
 150 };
 151
 152 #ifdef HAVE_LZMA
 153 /** Return an approximate number of bytes stored in memory to hold the LZMA
 154  * encoder/decoder state. */
 155 static size_t
 156 tor_lzma_state_size_precalc(int compress, compression_level_t level)
 157 {
 158   uint64_t memory_usage;
 159
 160   if (compress)
 161     memory_usage = lzma_easy_encoder_memusage(memory_level(level));
 162   else
 163     memory_usage = lzma_easy_decoder_memusage(memory_level(level));
 164
 165   if (memory_usage == UINT64_MAX) {
 166     // LCOV_EXCL_START
 167     log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s",
 168                          compress ? "encoder" : "decoder");
 169     goto err;
 170     // LCOV_EXCL_STOP
 171   }
 172
 173   if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX)
 174     memory_usage = SIZE_MAX;
 175   else
 176     memory_usage += sizeof(tor_lzma_compress_state_t);
 177
 178   return (size_t)memory_usage;
 179
 180  // LCOV_EXCL_START
 181  err:
 182   return 0;
 183  // LCOV_EXCL_STOP
 184 }
 185 #endif /* defined(HAVE_LZMA) */
 186
 187 /** Construct and return a tor_lzma_compress_state_t object using
 188  * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
 189  * decompression. */
 190 tor_lzma_compress_state_t *
 191 tor_lzma_compress_new(int compress,
 192                       compress_method_t method,
 193                       compression_level_t level)
 194 {
 195   tor_assert(method == LZMA_METHOD);
 196
 197 #ifdef HAVE_LZMA
 198   tor_lzma_compress_state_t *result;
 199   lzma_ret retval;
 200   lzma_options_lzma stream_options;
 201
 202   // Note that we do not explicitly initialize the lzma_stream object here,
 203   // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is
 204   // also what `tor_malloc_zero()` does.
 205   result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t));
 206   result->compress = compress;
 207   result->allocation = tor_lzma_state_size_precalc(compress, level);
 208
 209   if (compress) {
 210     lzma_lzma_preset(&stream_options, memory_level(level));
 211
 212     retval = lzma_alone_encoder(&result->stream, &stream_options);
 213
 214     if (retval != LZMA_OK) {
 215       // LCOV_EXCL_START
 216       log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
 217                lzma_error_str(retval), retval);
 218       goto err;
 219       // LCOV_EXCL_STOP
 220     }
 221   } else {
 222     retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT);
 223
 224     if (retval != LZMA_OK) {
 225       // LCOV_EXCL_START
 226       log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
 227                lzma_error_str(retval), retval);
 228       goto err;
 229       // LCOV_EXCL_STOP
 230     }
 231   }
 232
 233   atomic_counter_add(&total_lzma_allocation, result->allocation);
 234   return result;
 235
 236  /* LCOV_EXCL_START */
 237  err:
 238   tor_free(result);
 239   return NULL;
 240  /* LCOV_EXCL_STOP */
 241 #else /* !defined(HAVE_LZMA) */
 242   (void)compress;
 243   (void)method;
 244   (void)level;
 245
 246   return NULL;
 247 #endif /* defined(HAVE_LZMA) */
 248 }
 249
 250 /** Compress/decompress some bytes using <b>state</b>.  Read up to
 251  * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 252  * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 253  * we've reached the end of the input.
 254  *
 255  * Return TOR_COMPRESS_DONE if we've finished the entire
 256  * compression/decompression.
 257  * Return TOR_COMPRESS_OK if we're processed everything from the input.
 258  * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
 259  * Return TOR_COMPRESS_ERROR if the stream is corrupt.
 260  */
 261 tor_compress_output_t
 262 tor_lzma_compress_process(tor_lzma_compress_state_t *state,
 263                           char **out, size_t *out_len,
 264                           const char **in, size_t *in_len,
 265                           int finish)
 266 {
 267 #ifdef HAVE_LZMA
 268   lzma_ret retval;
 269   lzma_action action;
 270
 271   tor_assert(state != NULL);
 272   tor_assert(*in_len <= UINT_MAX);
 273   tor_assert(*out_len <= UINT_MAX);
 274
 275   state->stream.next_in = (unsigned char *)*in;
 276   state->stream.avail_in = *in_len;
 277   state->stream.next_out = (unsigned char *)*out;
 278   state->stream.avail_out = *out_len;
 279
 280   action = finish ? LZMA_FINISH : LZMA_RUN;
 281
 282   retval = lzma_code(&state->stream, action);
 283
 284   state->input_so_far += state->stream.next_in - ((unsigned char *)*in);
 285   state->output_so_far += state->stream.next_out - ((unsigned char *)*out);
 286
 287   *out = (char *)state->stream.next_out;
 288   *out_len = state->stream.avail_out;
 289   *in = (const char *)state->stream.next_in;
 290   *in_len = state->stream.avail_in;
 291
 292   if (! state->compress &&
 293       tor_compress_is_compression_bomb(state->input_so_far,
 294                                        state->output_so_far)) {
 295     log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
 296     return TOR_COMPRESS_ERROR;
 297   }
 298
 299   switch (retval) {
 300     case LZMA_OK:
 301       if (state->stream.avail_out == 0 || finish)
 302         return TOR_COMPRESS_BUFFER_FULL;
 303
 304       return TOR_COMPRESS_OK;
 305
 306     case LZMA_BUF_ERROR:
 307       if (state->stream.avail_in == 0 && !finish)
 308         return TOR_COMPRESS_OK;
 309
 310       return TOR_COMPRESS_BUFFER_FULL;
 311
 312     case LZMA_STREAM_END:
 313       return TOR_COMPRESS_DONE;
 314
 315     // We list all the possible values of `lzma_ret` here to silence the
 316     // `switch-enum` warning and to detect if a new member was added.
 317     case LZMA_NO_CHECK:
 318     case LZMA_UNSUPPORTED_CHECK:
 319     case LZMA_GET_CHECK:
 320     case LZMA_MEM_ERROR:
 321     case LZMA_MEMLIMIT_ERROR:
 322     case LZMA_FORMAT_ERROR:
 323     case LZMA_OPTIONS_ERROR:
 324     case LZMA_DATA_ERROR:
 325     case LZMA_PROG_ERROR:
 326 #if LZMA_VERSION >= 50030010
 327     case LZMA_SEEK_NEEDED:
 328 #endif
 329 #if LZMA_VERSION >= 50030020
 330     case LZMA_RET_INTERNAL1:
 331     case LZMA_RET_INTERNAL2:
 332     case LZMA_RET_INTERNAL3:
 333     case LZMA_RET_INTERNAL4:
 334     case LZMA_RET_INTERNAL5:
 335     case LZMA_RET_INTERNAL6:
 336     case LZMA_RET_INTERNAL7:
 337     case LZMA_RET_INTERNAL8:
 338 #endif
 339     default:
 340       log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.",
 341                state->compress ? "compression" : "decompression",
 342                lzma_error_str(retval));
 343       return TOR_COMPRESS_ERROR;
 344   }
 345 #else /* !defined(HAVE_LZMA) */
 346   (void)state;
 347   (void)out;
 348   (void)out_len;
 349   (void)in;
 350   (void)in_len;
 351   (void)finish;
 352   return TOR_COMPRESS_ERROR;
 353 #endif /* defined(HAVE_LZMA) */
 354 }
 355
 356 /** Deallocate <b>state</b>. */
 357 void
 358 tor_lzma_compress_free_(tor_lzma_compress_state_t *state)
 359 {
 360   if (state == NULL)
 361     return;
 362
 363   atomic_counter_sub(&total_lzma_allocation, state->allocation);
 364
 365 #ifdef HAVE_LZMA
 366   lzma_end(&state->stream);
 367 #endif
 368
 369   tor_free(state);
 370 }
 371
 372 /** Return the approximate number of bytes allocated for <b>state</b>. */
 373 size_t
 374 tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state)
 375 {
 376   tor_assert(state != NULL);
 377   return state->allocation;
 378 }
 379
 380 /** Return the approximate number of bytes allocated for all LZMA states. */
 381 size_t
 382 tor_lzma_get_total_allocation(void)
 383 {
 384   return atomic_counter_get(&total_lzma_allocation);
 385 }
 386
 387 /** Initialize the lzma module */
 388 void
 389 tor_lzma_init(void)
 390 {
 391   atomic_counter_init(&total_lzma_allocation);
 392 }