doc/examples/04_compress_easy_mt.c

   1 ///////////////////////////////////////////////////////////////////////////////
   2 //
   3 /// \file       04_compress_easy_mt.c
   4 /// \brief      Compress in multi-call mode using LZMA2 in multi-threaded mode
   5 ///
   6 /// Usage:      ./04_compress_easy_mt < INFILE > OUTFILE
   7 ///
   8 /// Example:    ./04_compress_easy_mt < foo > foo.xz
   9 //
  10 //  Author:     Lasse Collin
  11 //
  12 //  This file has been put into the public domain.
  13 //  You can do whatever you want with this file.
  14 //
  15 ///////////////////////////////////////////////////////////////////////////////
  16
  17 #include <stdbool.h>
  18 #include <stdlib.h>
  19 #include <stdio.h>
  20 #include <string.h>
  21 #include <errno.h>
  22 #include <lzma.h>
  23
  24
  25 static bool
  26 init_encoder(lzma_stream *strm)
  27 {
  28         // The threaded encoder takes the options as pointer to
  29         // a lzma_mt structure.
  30         lzma_mt mt = {
  31                 // No flags are needed.
  32                 .flags = 0,
  33
  34                 // Let liblzma determine a sane block size.
  35                 .block_size = 0,
  36
  37                 // Use no timeout for lzma_code() calls by setting timeout
  38                 // to zero. That is, sometimes lzma_code() might block for
  39                 // a long time (from several seconds to even minutes).
  40                 // If this is not OK, for example due to progress indicator
  41                 // needing updates, specify a timeout in milliseconds here.
  42                 // See the documentation of lzma_mt in lzma/container.h for
  43                 // information how to choose a reasonable timeout.
  44                 .timeout = 0,
  45
  46                 // Use the default preset (6) for LZMA2.
  47                 // To use a preset, filters must be set to NULL.
  48                 .preset = LZMA_PRESET_DEFAULT,
  49                 .filters = NULL,
  50
  51                 // Use CRC64 for integrity checking. See also
  52                 // 01_compress_easy.c about choosing the integrity check.
  53                 .check = LZMA_CHECK_CRC64,
  54         };
  55
  56         // Detect how many threads the CPU supports.
  57         mt.threads = lzma_cputhreads();
  58
  59         // If the number of CPU cores/threads cannot be detected,
  60         // use one thread. Note that this isn't the same as the normal
  61         // single-threaded mode as this will still split the data into
  62         // blocks and use more RAM than the normal single-threaded mode.
  63         // You may want to consider using lzma_easy_encoder() or
  64         // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
  65         // lzma_cputhreads() returns 0 or 1.
  66         if (mt.threads == 0)
  67                 mt.threads = 1;
  68
  69         // If the number of CPU cores/threads exceeds threads_max,
  70         // limit the number of threads to keep memory usage lower.
  71         // The number 8 is arbitrarily chosen and may be too low or
  72         // high depending on the compression preset and the computer
  73         // being used.
  74         //
  75         // FIXME: A better way could be to check the amount of RAM
  76         // (or available RAM) and use lzma_stream_encoder_mt_memusage()
  77         // to determine if the number of threads should be reduced.
  78         const uint32_t threads_max = 8;
  79         if (mt.threads > threads_max)
  80                 mt.threads = threads_max;
  81
  82         // Initialize the threaded encoder.
  83         lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
  84
  85         if (ret == LZMA_OK)
  86                 return true;
  87
  88         const char *msg;
  89         switch (ret) {
  90         case LZMA_MEM_ERROR:
  91                 msg = "Memory allocation failed";
  92                 break;
  93
  94         case LZMA_OPTIONS_ERROR:
  95                 // We are no longer using a plain preset so this error
  96                 // message has been edited accordingly compared to
  97                 // 01_compress_easy.c.
  98                 msg = "Specified filter chain is not supported";
  99                 break;
 100
 101         case LZMA_UNSUPPORTED_CHECK:
 102                 msg = "Specified integrity check is not supported";
 103                 break;
 104
 105         default:
 106                 msg = "Unknown error, possibly a bug";
 107                 break;
 108         }
 109
 110         fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
 111                         msg, ret);
 112         return false;
 113 }
 114
 115
 116 // This function is identical to the one in 01_compress_easy.c.
 117 static bool
 118 compress(lzma_stream *strm, FILE *infile, FILE *outfile)
 119 {
 120         lzma_action action = LZMA_RUN;
 121
 122         uint8_t inbuf[BUFSIZ];
 123         uint8_t outbuf[BUFSIZ];
 124
 125         strm->next_in = NULL;
 126         strm->avail_in = 0;
 127         strm->next_out = outbuf;
 128         strm->avail_out = sizeof(outbuf);
 129
 130         while (true) {
 131                 if (strm->avail_in == 0 && !feof(infile)) {
 132                         strm->next_in = inbuf;
 133                         strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
 134                                         infile);
 135
 136                         if (ferror(infile)) {
 137                                 fprintf(stderr, "Read error: %s\n",
 138                                                 strerror(errno));
 139                                 return false;
 140                         }
 141
 142                         if (feof(infile))
 143                                 action = LZMA_FINISH;
 144                 }
 145
 146                 lzma_ret ret = lzma_code(strm, action);
 147
 148                 if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
 149                         size_t write_size = sizeof(outbuf) - strm->avail_out;
 150
 151                         if (fwrite(outbuf, 1, write_size, outfile)
 152                                         != write_size) {
 153                                 fprintf(stderr, "Write error: %s\n",
 154                                                 strerror(errno));
 155                                 return false;
 156                         }
 157
 158                         strm->next_out = outbuf;
 159                         strm->avail_out = sizeof(outbuf);
 160                 }
 161
 162                 if (ret != LZMA_OK) {
 163                         if (ret == LZMA_STREAM_END)
 164                                 return true;
 165
 166                         const char *msg;
 167                         switch (ret) {
 168                         case LZMA_MEM_ERROR:
 169                                 msg = "Memory allocation failed";
 170                                 break;
 171
 172                         case LZMA_DATA_ERROR:
 173                                 msg = "File size limits exceeded";
 174                                 break;
 175
 176                         default:
 177                                 msg = "Unknown error, possibly a bug";
 178                                 break;
 179                         }
 180
 181                         fprintf(stderr, "Encoder error: %s (error code %u)\n",
 182                                         msg, ret);
 183                         return false;
 184                 }
 185         }
 186 }
 187
 188
 189 extern int
 190 main(void)
 191 {
 192         lzma_stream strm = LZMA_STREAM_INIT;
 193
 194         bool success = init_encoder(&strm);
 195         if (success)
 196                 success = compress(&strm, stdin, stdout);
 197
 198         lzma_end(&strm);
 199
 200         if (fclose(stdout)) {
 201                 fprintf(stderr, "Write error: %s\n", strerror(errno));
 202                 success = false;
 203         }
 204
 205         return success ? EXIT_SUCCESS : EXIT_FAILURE;
 206 }