dfltcc.c

   1 /* dfltcc.c -- compress data using IBM Z DEFLATE COMPRESSION CALL
   2
   3    Copyright (C) 2019-2022 Free Software Foundation, Inc.
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software Foundation,
  17    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
  18
  19 #include <config.h>
  20
  21 #include <stdalign.h>
  22 #include <stdbool.h>
  23 #include <stdlib.h>
  24
  25 #ifdef HAVE_SYS_SDT_H
  26 # include <sys/sdt.h>
  27 #endif
  28
  29 #include "tailor.h"
  30 #include "gzip.h"
  31
  32 #ifdef DYN_ALLOC
  33 # error "DYN_ALLOC is not supported by DFLTCC"
  34 #endif
  35
  36 /* ===========================================================================
  37  * C wrappers for the DEFLATE CONVERSION CALL instruction.
  38  */
  39
  40 typedef enum
  41 {
  42   DFLTCC_CC_OK = 0,
  43   DFLTCC_CC_OP1_TOO_SHORT = 1,
  44   DFLTCC_CC_OP2_TOO_SHORT = 2,
  45   DFLTCC_CC_OP2_CORRUPT = 2,
  46   DFLTCC_CC_AGAIN = 3,
  47 } dfltcc_cc;
  48
  49 #define DFLTCC_QAF 0
  50 #define DFLTCC_GDHT 1
  51 #define DFLTCC_CMPR 2
  52 #define DFLTCC_XPND 4
  53 #define HBT_CIRCULAR (1 << 7)
  54 #define DFLTCC_FACILITY 151
  55 #define DFLTCC_FMT0 0
  56 #define CVT_CRC32 0
  57 #define HTT_FIXED 0
  58 #define HTT_DYNAMIC 1
  59
  60 #ifndef DFLTCC_BLOCK_SIZE
  61 # define DFLTCC_BLOCK_SIZE 1048576
  62 #endif
  63 #ifndef DFLTCC_FIRST_FHT_BLOCK_SIZE
  64 # define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096
  65 #endif
  66 #ifndef DFLTCC_LEVEL_MASK
  67 # define DFLTCC_LEVEL_MASK 0x2
  68 #endif
  69 #ifndef DFLTCC_RIBM
  70 # define DFLTCC_RIBM 0
  71 #endif
  72
  73 #define MAX(a, b) ((a) > (b) ? (a) : (b))
  74
  75 #define STREQ(a, b) (strcmp (a, b) == 0)
  76
  77 struct dfltcc_qaf_param
  78 {
  79   char fns[16];
  80   char reserved1[8];
  81   char fmts[2];
  82   char reserved2[6];
  83 };
  84
  85 union aligned_dfltcc_qaf_param
  86 {
  87   struct dfltcc_qaf_param af;
  88   char alignas (8) aligned;
  89 };
  90
  91 struct dfltcc_param_v0
  92 {
  93   unsigned short pbvn;               /* Parameter-Block-Version Number */
  94   unsigned char mvn;                 /* Model-Version Number */
  95   unsigned char ribm;                /* Reserved for IBM use */
  96   unsigned reserved32 : 31;
  97   unsigned cf : 1;                   /* Continuation Flag */
  98   unsigned char reserved64[8];
  99   unsigned nt : 1;                   /* New Task */
 100   unsigned reserved129 : 1;
 101   unsigned cvt : 1;                  /* Check Value Type */
 102   unsigned reserved131 : 1;
 103   unsigned htt : 1;                  /* Huffman-Table Type */
 104   unsigned bcf : 1;                  /* Block-Continuation Flag */
 105   unsigned bcc : 1;                  /* Block Closing Control */
 106   unsigned bhf : 1;                  /* Block Header Final */
 107   unsigned reserved136 : 1;
 108   unsigned reserved137 : 1;
 109   unsigned dhtgc : 1;                /* DHT Generation Control */
 110   unsigned reserved139 : 5;
 111   unsigned reserved144 : 5;
 112   unsigned sbb : 3;                  /* Sub-Byte Boundary */
 113   unsigned char oesc;                /* Operation-Ending-Supplemental Code */
 114   unsigned reserved160 : 12;
 115   unsigned ifs : 4;                  /* Incomplete-Function Status */
 116   unsigned short ifl;                /* Incomplete-Function Length */
 117   unsigned char reserved192[8];
 118   unsigned char reserved256[8];
 119   unsigned char reserved320[4];
 120   unsigned short hl;                 /* History Length */
 121   unsigned reserved368 : 1;
 122   unsigned short ho : 15;            /* History Offset */
 123   unsigned int cv;                   /* Check Value */
 124   unsigned eobs : 15;                /* End-of-block Symbol */
 125   unsigned reserved431 : 1;
 126   unsigned char eobl : 4;            /* End-of-block Length */
 127   unsigned reserved436 : 12;
 128   unsigned reserved448 : 4;
 129   unsigned short cdhtl : 12;         /* Compressed-Dynamic-Huffman Table
 130                                         Length */
 131   unsigned char reserved464[6];
 132   unsigned char cdht[288];
 133   unsigned char reserved[32];
 134   unsigned char csb[1152];
 135 };
 136
 137 union aligned_dfltcc_param_v0
 138 {
 139   struct dfltcc_param_v0 param;
 140   char alignas (8) aligned;
 141 };
 142
 143 static int
 144 is_bit_set (const char *bits, int n)
 145 {
 146   return bits[n / 8] & (1 << (7 - (n % 8)));
 147 }
 148
 149 static int
 150 is_dfltcc_enabled (void)
 151 {
 152   char facilities[(DFLTCC_FACILITY / 64 + 1) * 8];
 153
 154   char const *env = getenv ("DFLTCC");
 155   if (env && STREQ (env, "0"))
 156     return 0;
 157
 158   memset (facilities, 0, sizeof facilities);
 159   register char r0 __asm__ ("r0") = sizeof facilities / 8 - 1;
 160   /* STFLE is supported since z9-109 and only in z/Architecture mode.  When
 161    * compiling with -m31, gcc defaults to ESA mode, however, since the kernel
 162    * is 64-bit, it's always z/Architecture mode at runtime.  */
 163   __asm__ (
 164 #ifndef __clang__
 165            ".machinemode push\n"
 166            ".machinemode zarch\n"
 167 #endif
 168            "stfle %[facilities]\n"
 169 #ifndef __clang__
 170            ".machinemode pop\n"
 171 #endif
 172            : [facilities] "=Q"(facilities), [r0] "+r"(r0) :: "cc");
 173   return is_bit_set (facilities, DFLTCC_FACILITY);
 174 }
 175
 176 static dfltcc_cc
 177 dfltcc (int fn, void *param,
 178         uch **op1, size_t *len1,
 179         uch const **op2, size_t *len2,
 180         void *hist)
 181 {
 182   uch *t2 = op1 ? *op1 : NULL;
 183   size_t t3 = len1 ? *len1 : 0;
 184   const uch *t4 = op2 ? *op2 : NULL;
 185   size_t t5 = len2 ? *len2 : 0;
 186   register int r0 __asm__ ("r0") = fn;
 187   register void *r1 __asm__ ("r1") = param;
 188   register uch *r2 __asm__ ("r2") = t2;
 189   register size_t r3 __asm__ ("r3") = t3;
 190   register const uch *r4 __asm__ ("r4") = t4;
 191   register size_t r5 __asm__ ("r5") = t5;
 192   int cc;
 193
 194   __asm__ volatile (
 195 #ifdef HAVE_SYS_SDT_H
 196                     STAP_PROBE_ASM (zlib, dfltcc_entry,
 197                                     STAP_PROBE_ASM_TEMPLATE (5))
 198 #endif
 199                     ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n"
 200 #ifdef HAVE_SYS_SDT_H
 201                     STAP_PROBE_ASM (zlib, dfltcc_exit,
 202                                     STAP_PROBE_ASM_TEMPLATE (5))
 203 #endif
 204                     "ipm %[cc]\n"
 205                     : [r2] "+r" (r2)
 206                       , [r3] "+r" (r3)
 207                       , [r4] "+r" (r4)
 208                       , [r5] "+r" (r5)
 209                       , [cc] "=r" (cc)
 210                     : [r0] "r" (r0)
 211                       , [r1] "r" (r1)
 212                       , [hist] "r" (hist)
 213 #ifdef HAVE_SYS_SDT_H
 214                       , STAP_PROBE_ASM_OPERANDS (5, r2, r3, r4, r5, hist)
 215 #endif
 216                     : "cc", "memory");
 217   t2 = r2; t3 = r3; t4 = r4; t5 = r5;
 218
 219   if (op1)
 220     *op1 = t2;
 221   if (len1)
 222     *len1 = t3;
 223   if (op2)
 224     *op2 = t4;
 225   if (len2)
 226     *len2 = t5;
 227   return (cc >> 28) & 3;
 228 }
 229
 230 static void
 231 dfltcc_qaf (struct dfltcc_qaf_param *param)
 232 {
 233   dfltcc (DFLTCC_QAF, param, NULL, NULL, NULL, NULL, NULL);
 234 }
 235
 236 static void
 237 dfltcc_gdht (struct dfltcc_param_v0 *param)
 238 {
 239   const uch *next_in = inbuf + inptr;
 240   size_t avail_in = insize - inptr;
 241
 242   dfltcc (DFLTCC_GDHT, param, NULL, NULL, &next_in, &avail_in, NULL);
 243 }
 244
 245 static dfltcc_cc
 246 dfltcc_cmpr_xpnd (struct dfltcc_param_v0 *param, int fn, off_t *total_in)
 247 {
 248   uch *next_out = outbuf + outcnt;
 249   size_t avail_out = OUTBUFSIZ - outcnt;
 250   const uch *next_in = inbuf + inptr;
 251   size_t avail_in = insize - inptr;
 252   dfltcc_cc cc = dfltcc (fn | HBT_CIRCULAR, param,
 253                          &next_out, &avail_out,
 254                          &next_in, &avail_in,
 255                          window);
 256   off_t consumed_in = next_in - (inbuf + inptr);
 257   inptr += consumed_in;
 258   *total_in += consumed_in;
 259   outcnt += ((OUTBUFSIZ - outcnt) - avail_out);
 260   return cc;
 261 }
 262
 263 static struct dfltcc_param_v0 *
 264 init_param (union aligned_dfltcc_param_v0 *ctx)
 265 {
 266   char const *s = getenv ("DFLTCC_RIBM");
 267   struct dfltcc_param_v0 *param = &ctx->param;
 268   memset (param, 0, sizeof *param);
 269   param->ribm = s && *s ? strtoul (s, NULL, 0) : DFLTCC_RIBM;
 270   param->nt = 1;
 271   param->cvt = CVT_CRC32;
 272   param->cv = __builtin_bswap32 (getcrc ());
 273   return param;
 274 }
 275
 276 static void
 277 bi_load (struct dfltcc_param_v0 *param)
 278 {
 279   bi_valid = param->sbb;
 280   bi_buf = bi_valid == 0 ? 0 : outbuf[outcnt] & ((1 << bi_valid) - 1);
 281 }
 282
 283 static void
 284 bi_close_block (struct dfltcc_param_v0 *param)
 285 {
 286   bi_load (param);
 287   send_bits (bi_reverse (param->eobs >> (15 - param->eobl), param->eobl),
 288              param->eobl);
 289   param->bcf = 0;
 290 }
 291
 292 static void
 293 close_block (struct dfltcc_param_v0 *param)
 294 {
 295   bi_close_block (param);
 296   bi_windup ();
 297   /* bi_windup has written out a possibly partial byte, fix up the position */
 298   param->sbb = (param->sbb + param->eobl) % 8;
 299   if (param->sbb != 0)
 300     {
 301       Assert (outcnt > 0, "outbuf must have enough space for EOBS");
 302       outcnt--;
 303     }
 304 }
 305
 306 static void
 307 close_stream (struct dfltcc_param_v0 *param)
 308 {
 309   if (param->bcf)
 310     bi_close_block (param);
 311   else
 312     bi_load (param);
 313   send_bits (1, 3); /* BFINAL=1, BTYPE=00 */
 314   bi_windup ();
 315   put_short (0x0000);
 316   put_short (0xFFFF);
 317 }
 318
 319 /* Compress ifd into ofd in hardware or fall back to software.  */
 320
 321 int
 322 dfltcc_deflate (int pack_level)
 323 {
 324   /* Check whether we can use hardware compression.  */
 325   if (!is_dfltcc_enabled () || getenv ("SOURCE_DATE_EPOCH"))
 326     return deflate (pack_level);
 327   char const *s = getenv ("DFLTCC_LEVEL_MASK");
 328   unsigned long level_mask
 329     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_LEVEL_MASK;
 330   if ((level_mask & (1 << pack_level)) == 0)
 331     return deflate (pack_level);
 332   union aligned_dfltcc_qaf_param ctx;
 333   dfltcc_qaf (&ctx.af);
 334   if (!is_bit_set (ctx.af.fns, DFLTCC_CMPR)
 335       || !is_bit_set (ctx.af.fns, DFLTCC_GDHT)
 336       || !is_bit_set (ctx.af.fmts, DFLTCC_FMT0))
 337     return deflate (pack_level);
 338
 339   /* Initialize tuning parameters.  */
 340   s = getenv ("DFLTCC_BLOCK_SIZE");
 341   unsigned long block_size
 342     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_BLOCK_SIZE;
 343
 344   s = getenv ("DFLTCC_FIRST_FHT_BLOCK_SIZE");
 345   off_t block_threshold
 346     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_FIRST_FHT_BLOCK_SIZE;
 347
 348   union aligned_dfltcc_param_v0 ctx_v0;
 349   struct dfltcc_param_v0 *param = init_param (&ctx_v0);
 350   off_t total_in = 0;
 351
 352   /* Compress ifd into ofd in a loop.  */
 353   while (true)
 354     {
 355       /* Flush the output data.  */
 356       if (outcnt > OUTBUFSIZ - 8)
 357         {
 358           if (param->sbb == 0)
 359             flush_outbuf ();
 360           else
 361             {
 362               uch partial = outbuf[outcnt];
 363               flush_outbuf ();
 364               outbuf[outcnt] = partial;
 365             }
 366         }
 367
 368       /* Close the block.  */
 369       if (param->bcf && total_in == block_threshold && !param->cf)
 370         {
 371           close_block (param);
 372           block_threshold += block_size;
 373         }
 374
 375       /* Read the input data.  */
 376       if (inptr == insize)
 377         {
 378           if (fill_inbuf (1) == EOF && !param->cf)
 379             break;
 380           inptr = 0;
 381         }
 382
 383       /* Temporarily mask some input data.  */
 384       int extra = MAX (0, total_in + (insize - inptr) - block_threshold);
 385       insize -= extra;
 386
 387       /* Start a new block.  */
 388       if (!param->bcf)
 389         {
 390           if (total_in == 0 && block_threshold > 0)
 391             param->htt = HTT_FIXED;
 392           else
 393             {
 394               param->htt = HTT_DYNAMIC;
 395               dfltcc_gdht (param);
 396             }
 397         }
 398
 399       /* Compress inbuf into outbuf.  */
 400       while (dfltcc_cmpr_xpnd (param, DFLTCC_CMPR, &total_in)
 401              == DFLTCC_CC_AGAIN)
 402         ;
 403
 404       /* Unmask the input data.  */
 405       insize += extra;
 406
 407       /* Continue the block */
 408       param->bcf = 1;
 409     }
 410
 411   close_stream (param);
 412   setcrc (__builtin_bswap32 (param->cv));
 413   return 0;
 414 }
 415
 416 /* Decompress ifd into ofd in hardware or fall back to software.  */
 417 int
 418 dfltcc_inflate (void)
 419 {
 420   /* Check whether we can use hardware decompression.  */
 421   if (!is_dfltcc_enabled ())
 422     return inflate ();
 423   union aligned_dfltcc_qaf_param ctx;
 424   dfltcc_qaf (&ctx.af);
 425   if (!is_bit_set (ctx.af.fns, DFLTCC_XPND))
 426     return inflate ();
 427
 428   union aligned_dfltcc_param_v0 ctx_v0;
 429   struct dfltcc_param_v0 *param = init_param (&ctx_v0);
 430   off_t total_in = 0;
 431
 432   /* Decompress ifd into ofd in a loop.  */
 433   while (true)
 434     {
 435       /* Perform I/O.  */
 436       if (outcnt == OUTBUFSIZ)
 437         flush_outbuf ();
 438       if (inptr == insize)
 439         {
 440           if (fill_inbuf (1) == EOF)
 441             {
 442               /* Premature EOF.  */
 443               return 2;
 444             }
 445           inptr = 0;
 446         }
 447
 448         /* Decompress inbuf into outbuf.  */
 449         dfltcc_cc cc;
 450         while ((cc = dfltcc_cmpr_xpnd (param, DFLTCC_XPND, &total_in))
 451                == DFLTCC_CC_AGAIN)
 452           ;
 453         if (cc == DFLTCC_CC_OK)
 454           {
 455             /* The entire deflate stream has been successfully decompressed.  */
 456             break;
 457           }
 458         if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0)
 459           {
 460             /* The deflate stream is corrupted.  */
 461             fprintf (stderr, "Operation-Ending-Supplemental Code 0x%x\n",
 462                      param->oesc);
 463             flush_outbuf ();
 464             return 2;
 465           }
 466         /* There must be more data to decompress.  */
 467     }
 468
 469   if (param->sbb != 0)
 470     {
 471       /* The deflate stream has ended in the middle of a byte.  Go to
 472          the next byte boundary, so that unzip can read CRC and length.  */
 473       inptr++;
 474     }
 475
 476   /* Set CRC value and update bytes_out for unzip.  */
 477   setcrc (__builtin_bswap32 (param->cv));
 478   flush_outbuf ();
 479   return 0;
 480 }