dfltcc.c

   1 /* dfltcc.c -- compress data using IBM Z DEFLATE COMPRESSION CALL
   2
   3    Copyright (C) 2019-2024 Free Software Foundation, Inc.
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software Foundation,
  17    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
  18
  19 #include <config.h>
  20
  21 #include <stdlib.h>
  22
  23 #ifdef HAVE_SYS_SDT_H
  24 # include <sys/sdt.h>
  25 #endif
  26
  27 #include "tailor.h"
  28 #include "gzip.h"
  29
  30 #ifdef DYN_ALLOC
  31 # error "DYN_ALLOC is not supported by DFLTCC"
  32 #endif
  33
  34 /* ===========================================================================
  35  * C wrappers for the DEFLATE CONVERSION CALL instruction.
  36  */
  37
  38 typedef enum
  39 {
  40   DFLTCC_CC_OK = 0,
  41   DFLTCC_CC_OP1_TOO_SHORT = 1,
  42   DFLTCC_CC_OP2_TOO_SHORT = 2,
  43   DFLTCC_CC_OP2_CORRUPT = 2,
  44   DFLTCC_CC_AGAIN = 3,
  45 } dfltcc_cc;
  46
  47 #define DFLTCC_QAF 0
  48 #define DFLTCC_GDHT 1
  49 #define DFLTCC_CMPR 2
  50 #define DFLTCC_XPND 4
  51 #define HBT_CIRCULAR (1 << 7)
  52 #define DFLTCC_FACILITY 151
  53 #define DFLTCC_FMT0 0
  54 #define CVT_CRC32 0
  55 #define HTT_FIXED 0
  56 #define HTT_DYNAMIC 1
  57
  58 #ifndef DFLTCC_BLOCK_SIZE
  59 # define DFLTCC_BLOCK_SIZE 1048576
  60 #endif
  61 #ifndef DFLTCC_FIRST_FHT_BLOCK_SIZE
  62 # define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096
  63 #endif
  64 #ifndef DFLTCC_LEVEL_MASK
  65 # define DFLTCC_LEVEL_MASK 0x2
  66 #endif
  67 #ifndef DFLTCC_RIBM
  68 # define DFLTCC_RIBM 0
  69 #endif
  70
  71 #define MAX(a, b) ((a) > (b) ? (a) : (b))
  72
  73 #define STREQ(a, b) (strcmp (a, b) == 0)
  74
  75 struct dfltcc_qaf_param
  76 {
  77   char fns[16];
  78   char reserved1[8];
  79   char fmts[2];
  80   char reserved2[6];
  81 };
  82
  83 union aligned_dfltcc_qaf_param
  84 {
  85   struct dfltcc_qaf_param af;
  86   char alignas (8) aligned;
  87 };
  88
  89 struct dfltcc_param_v0
  90 {
  91   unsigned short pbvn;               /* Parameter-Block-Version Number */
  92   unsigned char mvn;                 /* Model-Version Number */
  93   unsigned char ribm;                /* Reserved for IBM use */
  94   unsigned reserved32 : 31;
  95   unsigned cf : 1;                   /* Continuation Flag */
  96   unsigned char reserved64[8];
  97   unsigned nt : 1;                   /* New Task */
  98   unsigned reserved129 : 1;
  99   unsigned cvt : 1;                  /* Check Value Type */
 100   unsigned reserved131 : 1;
 101   unsigned htt : 1;                  /* Huffman-Table Type */
 102   unsigned bcf : 1;                  /* Block-Continuation Flag */
 103   unsigned bcc : 1;                  /* Block Closing Control */
 104   unsigned bhf : 1;                  /* Block Header Final */
 105   unsigned reserved136 : 1;
 106   unsigned reserved137 : 1;
 107   unsigned dhtgc : 1;                /* DHT Generation Control */
 108   unsigned reserved139 : 5;
 109   unsigned reserved144 : 5;
 110   unsigned sbb : 3;                  /* Sub-Byte Boundary */
 111   unsigned char oesc;                /* Operation-Ending-Supplemental Code */
 112   unsigned reserved160 : 12;
 113   unsigned ifs : 4;                  /* Incomplete-Function Status */
 114   unsigned short ifl;                /* Incomplete-Function Length */
 115   unsigned char reserved192[8];
 116   unsigned char reserved256[8];
 117   unsigned char reserved320[4];
 118   unsigned short hl;                 /* History Length */
 119   unsigned reserved368 : 1;
 120   unsigned short ho : 15;            /* History Offset */
 121   unsigned int cv;                   /* Check Value */
 122   unsigned eobs : 15;                /* End-of-block Symbol */
 123   unsigned reserved431 : 1;
 124   unsigned char eobl : 4;            /* End-of-block Length */
 125   unsigned reserved436 : 12;
 126   unsigned reserved448 : 4;
 127   unsigned short cdhtl : 12;         /* Compressed-Dynamic-Huffman Table
 128                                         Length */
 129   unsigned char reserved464[6];
 130   unsigned char cdht[288];
 131   unsigned char reserved[32];
 132   unsigned char csb[1152];
 133 };
 134
 135 union aligned_dfltcc_param_v0
 136 {
 137   struct dfltcc_param_v0 param;
 138   char alignas (8) aligned;
 139 };
 140
 141 static int
 142 is_bit_set (const char *bits, int n)
 143 {
 144   return bits[n / 8] & (1 << (7 - (n % 8)));
 145 }
 146
 147 static int
 148 is_dfltcc_enabled (void)
 149 {
 150   char facilities[(DFLTCC_FACILITY / 64 + 1) * 8];
 151
 152   char const *env = getenv ("DFLTCC");
 153   if (env && STREQ (env, "0"))
 154     return 0;
 155
 156   memset (facilities, 0, sizeof facilities);
 157   register char r0 __asm__ ("r0") = sizeof facilities / 8 - 1;
 158   /* STFLE is supported since z9-109 and only in z/Architecture mode.  When
 159    * compiling with -m31, gcc defaults to ESA mode, however, since the kernel
 160    * is 64-bit, it's always z/Architecture mode at runtime.  */
 161   __asm__ (
 162 #ifndef __clang__
 163            ".machinemode push\n"
 164            ".machinemode zarch\n"
 165 #endif
 166            "stfle %[facilities]\n"
 167 #ifndef __clang__
 168            ".machinemode pop\n"
 169 #endif
 170            : [facilities] "=Q"(facilities), [r0] "+r"(r0) :: "cc");
 171   return is_bit_set (facilities, DFLTCC_FACILITY);
 172 }
 173
 174 static dfltcc_cc
 175 dfltcc (int fn, void *param,
 176         uch **op1, size_t *len1,
 177         uch const **op2, size_t *len2,
 178         void *hist)
 179 {
 180   uch *t2 = op1 ? *op1 : NULL;
 181   size_t t3 = len1 ? *len1 : 0;
 182   const uch *t4 = op2 ? *op2 : NULL;
 183   size_t t5 = len2 ? *len2 : 0;
 184   register int r0 __asm__ ("r0") = fn;
 185   register void *r1 __asm__ ("r1") = param;
 186   register uch *r2 __asm__ ("r2") = t2;
 187   register size_t r3 __asm__ ("r3") = t3;
 188   register const uch *r4 __asm__ ("r4") = t4;
 189   register size_t r5 __asm__ ("r5") = t5;
 190   int cc;
 191
 192   __asm__ volatile (
 193 #ifdef HAVE_SYS_SDT_H
 194                     STAP_PROBE_ASM (zlib, dfltcc_entry,
 195                                     STAP_PROBE_ASM_TEMPLATE (5))
 196 #endif
 197                     ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n"
 198 #ifdef HAVE_SYS_SDT_H
 199                     STAP_PROBE_ASM (zlib, dfltcc_exit,
 200                                     STAP_PROBE_ASM_TEMPLATE (5))
 201 #endif
 202                     "ipm %[cc]\n"
 203                     : [r2] "+r" (r2)
 204                       , [r3] "+r" (r3)
 205                       , [r4] "+r" (r4)
 206                       , [r5] "+r" (r5)
 207                       , [cc] "=r" (cc)
 208                     : [r0] "r" (r0)
 209                       , [r1] "r" (r1)
 210                       , [hist] "r" (hist)
 211 #ifdef HAVE_SYS_SDT_H
 212                       , STAP_PROBE_ASM_OPERANDS (5, r2, r3, r4, r5, hist)
 213 #endif
 214                     : "cc", "memory");
 215   t2 = r2; t3 = r3; t4 = r4; t5 = r5;
 216
 217   if (op1)
 218     *op1 = t2;
 219   if (len1)
 220     *len1 = t3;
 221   if (op2)
 222     *op2 = t4;
 223   if (len2)
 224     *len2 = t5;
 225   return (cc >> 28) & 3;
 226 }
 227
 228 static void
 229 dfltcc_qaf (struct dfltcc_qaf_param *param)
 230 {
 231   dfltcc (DFLTCC_QAF, param, NULL, NULL, NULL, NULL, NULL);
 232 }
 233
 234 static void
 235 dfltcc_gdht (struct dfltcc_param_v0 *param)
 236 {
 237   const uch *next_in = inbuf + inptr;
 238   size_t avail_in = insize - inptr;
 239
 240   dfltcc (DFLTCC_GDHT, param, NULL, NULL, &next_in, &avail_in, NULL);
 241 }
 242
 243 static dfltcc_cc
 244 dfltcc_cmpr_xpnd (struct dfltcc_param_v0 *param, int fn, off_t *total_in)
 245 {
 246   uch *next_out = outbuf + outcnt;
 247   size_t avail_out = OUTBUFSIZ - outcnt;
 248   const uch *next_in = inbuf + inptr;
 249   size_t avail_in = insize - inptr;
 250   dfltcc_cc cc = dfltcc (fn | HBT_CIRCULAR, param,
 251                          &next_out, &avail_out,
 252                          &next_in, &avail_in,
 253                          window);
 254   off_t consumed_in = next_in - (inbuf + inptr);
 255   inptr += consumed_in;
 256   *total_in += consumed_in;
 257   outcnt += ((OUTBUFSIZ - outcnt) - avail_out);
 258   return cc;
 259 }
 260
 261 static struct dfltcc_param_v0 *
 262 init_param (union aligned_dfltcc_param_v0 *ctx)
 263 {
 264   char const *s = getenv ("DFLTCC_RIBM");
 265   struct dfltcc_param_v0 *param = &ctx->param;
 266   memset (param, 0, sizeof *param);
 267   param->ribm = s && *s ? strtoul (s, NULL, 0) : DFLTCC_RIBM;
 268   param->nt = 1;
 269   param->cvt = CVT_CRC32;
 270   param->cv = __builtin_bswap32 (getcrc ());
 271   return param;
 272 }
 273
 274 static void
 275 bi_load (struct dfltcc_param_v0 *param)
 276 {
 277   bi_valid = param->sbb;
 278   bi_buf = bi_valid == 0 ? 0 : outbuf[outcnt] & ((1 << bi_valid) - 1);
 279 }
 280
 281 static void
 282 bi_close_block (struct dfltcc_param_v0 *param)
 283 {
 284   bi_load (param);
 285   send_bits (bi_reverse (param->eobs >> (15 - param->eobl), param->eobl),
 286              param->eobl);
 287   param->bcf = 0;
 288 }
 289
 290 static void
 291 close_block (struct dfltcc_param_v0 *param)
 292 {
 293   bi_close_block (param);
 294   bi_windup ();
 295   /* bi_windup has written out a possibly partial byte, fix up the position */
 296   param->sbb = (param->sbb + param->eobl) % 8;
 297   if (param->sbb != 0)
 298     {
 299       Assert (outcnt > 0, "outbuf must have enough space for EOBS");
 300       outcnt--;
 301     }
 302 }
 303
 304 static void
 305 close_stream (struct dfltcc_param_v0 *param)
 306 {
 307   if (param->bcf)
 308     bi_close_block (param);
 309   else
 310     bi_load (param);
 311   send_bits (1, 3); /* BFINAL=1, BTYPE=00 */
 312   bi_windup ();
 313   put_short (0x0000);
 314   put_short (0xFFFF);
 315 }
 316
 317 /* Compress ifd into ofd in hardware or fall back to software.  */
 318
 319 int
 320 dfltcc_deflate (int pack_level)
 321 {
 322   /* Check whether we can use hardware compression.  */
 323   if (!is_dfltcc_enabled () || getenv ("SOURCE_DATE_EPOCH"))
 324     return gzip_deflate (pack_level);
 325   char const *s = getenv ("DFLTCC_LEVEL_MASK");
 326   unsigned long level_mask
 327     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_LEVEL_MASK;
 328   if ((level_mask & (1 << pack_level)) == 0)
 329     return gzip_deflate (pack_level);
 330   union aligned_dfltcc_qaf_param ctx;
 331   dfltcc_qaf (&ctx.af);
 332   if (!is_bit_set (ctx.af.fns, DFLTCC_CMPR)
 333       || !is_bit_set (ctx.af.fns, DFLTCC_GDHT)
 334       || !is_bit_set (ctx.af.fmts, DFLTCC_FMT0))
 335     return gzip_deflate (pack_level);
 336
 337   /* Initialize tuning parameters.  */
 338   s = getenv ("DFLTCC_BLOCK_SIZE");
 339   unsigned long block_size
 340     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_BLOCK_SIZE;
 341
 342   s = getenv ("DFLTCC_FIRST_FHT_BLOCK_SIZE");
 343   off_t block_threshold
 344     = s && *s ? strtoul (s, NULL, 0) : DFLTCC_FIRST_FHT_BLOCK_SIZE;
 345
 346   union aligned_dfltcc_param_v0 ctx_v0;
 347   struct dfltcc_param_v0 *param = init_param (&ctx_v0);
 348   off_t total_in = 0;
 349
 350   /* Compress ifd into ofd in a loop.  */
 351   while (true)
 352     {
 353       /* Flush the output data.  */
 354       if (outcnt > OUTBUFSIZ - 8)
 355         {
 356           if (param->sbb == 0)
 357             flush_outbuf ();
 358           else
 359             {
 360               uch partial = outbuf[outcnt];
 361               flush_outbuf ();
 362               outbuf[outcnt] = partial;
 363             }
 364         }
 365
 366       /* Close the block.  */
 367       if (param->bcf && total_in == block_threshold && !param->cf)
 368         {
 369           close_block (param);
 370           block_threshold += block_size;
 371         }
 372
 373       /* Read the input data.  */
 374       if (inptr == insize)
 375         {
 376           if (fill_inbuf (1) == EOF && !param->cf)
 377             break;
 378           inptr = 0;
 379         }
 380
 381       /* Temporarily mask some input data.  */
 382       int extra = MAX (0, total_in + (insize - inptr) - block_threshold);
 383       insize -= extra;
 384
 385       /* Start a new block.  */
 386       if (!param->bcf)
 387         {
 388           if (total_in == 0 && block_threshold > 0)
 389             param->htt = HTT_FIXED;
 390           else
 391             {
 392               param->htt = HTT_DYNAMIC;
 393               dfltcc_gdht (param);
 394             }
 395         }
 396
 397       /* Compress inbuf into outbuf.  */
 398       while (dfltcc_cmpr_xpnd (param, DFLTCC_CMPR, &total_in)
 399              == DFLTCC_CC_AGAIN)
 400         ;
 401
 402       /* Unmask the input data.  */
 403       insize += extra;
 404
 405       /* Continue the block */
 406       param->bcf = 1;
 407     }
 408
 409   close_stream (param);
 410   setcrc (__builtin_bswap32 (param->cv));
 411   return 0;
 412 }
 413
 414 /* Decompress ifd into ofd in hardware or fall back to software.  */
 415 int
 416 dfltcc_inflate ()
 417 {
 418   /* Check whether we can use hardware decompression.  */
 419   if (!is_dfltcc_enabled ())
 420     return gzip_inflate ();
 421   union aligned_dfltcc_qaf_param ctx;
 422   dfltcc_qaf (&ctx.af);
 423   if (!is_bit_set (ctx.af.fns, DFLTCC_XPND))
 424     return gzip_inflate ();
 425
 426   union aligned_dfltcc_param_v0 ctx_v0;
 427   struct dfltcc_param_v0 *param = init_param (&ctx_v0);
 428   off_t total_in = 0;
 429
 430   /* Decompress ifd into ofd in a loop.  */
 431   while (true)
 432     {
 433       /* Perform I/O.  */
 434       if (outcnt == OUTBUFSIZ)
 435         flush_outbuf ();
 436       if (inptr == insize)
 437         {
 438           if (fill_inbuf (1) == EOF)
 439             {
 440               /* Premature EOF.  */
 441               return 2;
 442             }
 443           inptr = 0;
 444         }
 445
 446         /* Decompress inbuf into outbuf.  */
 447         dfltcc_cc cc;
 448         while ((cc = dfltcc_cmpr_xpnd (param, DFLTCC_XPND, &total_in))
 449                == DFLTCC_CC_AGAIN)
 450           ;
 451         if (cc == DFLTCC_CC_OK)
 452           {
 453             /* The entire deflate stream has been successfully decompressed.  */
 454             break;
 455           }
 456         if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0)
 457           {
 458             /* The deflate stream is corrupted.  */
 459             fprintf (stderr, "Operation-Ending-Supplemental Code 0x%x\n",
 460                      param->oesc);
 461             flush_outbuf ();
 462             return 2;
 463           }
 464         /* There must be more data to decompress.  */
 465     }
 466
 467   if (param->sbb != 0)
 468     {
 469       /* The deflate stream has ended in the middle of a byte.  Go to
 470          the next byte boundary, so that unzip can read CRC and length.  */
 471       inptr++;
 472     }
 473
 474   /* Set CRC value and update bytes_out for unzip.  */
 475   setcrc (__builtin_bswap32 (param->cv));
 476   flush_outbuf ();
 477   return 0;
 478 }