lib/Target/X86/X86ShuffleDecodeConstantPool.cpp

   1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Define several functions to decode x86 specific shuffle semantics using
  10 // constants from the constant pool.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "Utils/X86ShuffleDecode.h"
  15 #include "llvm/ADT/APInt.h"
  16 #include "llvm/IR/Constants.h"
  17
  18 //===----------------------------------------------------------------------===//
  19 //  Vector Mask Decoding
  20 //===----------------------------------------------------------------------===//
  21
  22 namespace llvm {
  23
  24 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
  25                                 APInt &UndefElts,
  26                                 SmallVectorImpl<uint64_t> &RawMask) {
  27   // It is not an error for shuffle masks to not be a vector of
  28   // MaskEltSizeInBits because the constant pool uniques constants by their
  29   // bit representation.
  30   // e.g. the following take up the same space in the constant pool:
  31   //   i128 -170141183420855150465331762880109871104
  32   //
  33   //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
  34   //
  35   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
  36   //              i32 -2147483648, i32 -2147483648>
  37   Type *CstTy = C->getType();
  38   if (!CstTy->isVectorTy())
  39     return false;
  40
  41   Type *CstEltTy = CstTy->getVectorElementType();
  42   if (!CstEltTy->isIntegerTy())
  43     return false;
  44
  45   unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
  46   unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
  47   unsigned NumCstElts = CstTy->getVectorNumElements();
  48
  49   assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
  50          "Unaligned shuffle mask size");
  51
  52   unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
  53   UndefElts = APInt(NumMaskElts, 0);
  54   RawMask.resize(NumMaskElts, 0);
  55
  56   // Fast path - if the constants match the mask size then copy direct.
  57   if (MaskEltSizeInBits == CstEltSizeInBits) {
  58     assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
  59     for (unsigned i = 0; i != NumMaskElts; ++i) {
  60       Constant *COp = C->getAggregateElement(i);
  61       if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
  62         return false;
  63
  64       if (isa<UndefValue>(COp)) {
  65         UndefElts.setBit(i);
  66         RawMask[i] = 0;
  67         continue;
  68       }
  69
  70       auto *Elt = cast<ConstantInt>(COp);
  71       RawMask[i] = Elt->getValue().getZExtValue();
  72     }
  73     return true;
  74   }
  75
  76   // Extract all the undef/constant element data and pack into single bitsets.
  77   APInt UndefBits(CstSizeInBits, 0);
  78   APInt MaskBits(CstSizeInBits, 0);
  79   for (unsigned i = 0; i != NumCstElts; ++i) {
  80     Constant *COp = C->getAggregateElement(i);
  81     if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
  82       return false;
  83
  84     unsigned BitOffset = i * CstEltSizeInBits;
  85
  86     if (isa<UndefValue>(COp)) {
  87       UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
  88       continue;
  89     }
  90
  91     MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
  92   }
  93
  94   // Now extract the undef/constant bit data into the raw shuffle masks.
  95   for (unsigned i = 0; i != NumMaskElts; ++i) {
  96     unsigned BitOffset = i * MaskEltSizeInBits;
  97     APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
  98
  99     // Only treat the element as UNDEF if all bits are UNDEF, otherwise
 100     // treat it as zero.
 101     if (EltUndef.isAllOnesValue()) {
 102       UndefElts.setBit(i);
 103       RawMask[i] = 0;
 104       continue;
 105     }
 106
 107     APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
 108     RawMask[i] = EltBits.getZExtValue();
 109   }
 110
 111   return true;
 112 }
 113
 114 void DecodePSHUFBMask(const Constant *C, unsigned Width,
 115                       SmallVectorImpl<int> &ShuffleMask) {
 116   assert((Width == 128 || Width == 256 || Width == 512) &&
 117          C->getType()->getPrimitiveSizeInBits() >= Width &&
 118          "Unexpected vector size.");
 119
 120   // The shuffle mask requires a byte vector.
 121   APInt UndefElts;
 122   SmallVector<uint64_t, 64> RawMask;
 123   if (!extractConstantMask(C, 8, UndefElts, RawMask))
 124     return;
 125
 126   unsigned NumElts = Width / 8;
 127   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
 128          "Unexpected number of vector elements.");
 129
 130   for (unsigned i = 0; i != NumElts; ++i) {
 131     if (UndefElts[i]) {
 132       ShuffleMask.push_back(SM_SentinelUndef);
 133       continue;
 134     }
 135
 136     uint64_t Element = RawMask[i];
 137     // If the high bit (7) of the byte is set, the element is zeroed.
 138     if (Element & (1 << 7))
 139       ShuffleMask.push_back(SM_SentinelZero);
 140     else {
 141       // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
 142       // lane of the vector we're inside.
 143       unsigned Base = i & ~0xf;
 144
 145       // Only the least significant 4 bits of the byte are used.
 146       int Index = Base + (Element & 0xf);
 147       ShuffleMask.push_back(Index);
 148     }
 149   }
 150 }
 151
 152 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
 153                         SmallVectorImpl<int> &ShuffleMask) {
 154   assert((Width == 128 || Width == 256 || Width == 512) &&
 155          C->getType()->getPrimitiveSizeInBits() >= Width &&
 156          "Unexpected vector size.");
 157   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
 158
 159   // The shuffle mask requires elements the same size as the target.
 160   APInt UndefElts;
 161   SmallVector<uint64_t, 16> RawMask;
 162   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
 163     return;
 164
 165   unsigned NumElts = Width / ElSize;
 166   unsigned NumEltsPerLane = 128 / ElSize;
 167   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
 168          "Unexpected number of vector elements.");
 169
 170   for (unsigned i = 0; i != NumElts; ++i) {
 171     if (UndefElts[i]) {
 172       ShuffleMask.push_back(SM_SentinelUndef);
 173       continue;
 174     }
 175
 176     int Index = i & ~(NumEltsPerLane - 1);
 177     uint64_t Element = RawMask[i];
 178     if (ElSize == 64)
 179       Index += (Element >> 1) & 0x1;
 180     else
 181       Index += Element & 0x3;
 182
 183     ShuffleMask.push_back(Index);
 184   }
 185 }
 186
 187 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
 188                          unsigned Width,
 189                          SmallVectorImpl<int> &ShuffleMask) {
 190   Type *MaskTy = C->getType();
 191   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
 192   (void)MaskTySize;
 193   assert((MaskTySize == 128 || MaskTySize == 256) &&
 194          Width >= MaskTySize && "Unexpected vector size.");
 195
 196   // The shuffle mask requires elements the same size as the target.
 197   APInt UndefElts;
 198   SmallVector<uint64_t, 8> RawMask;
 199   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
 200     return;
 201
 202   unsigned NumElts = Width / ElSize;
 203   unsigned NumEltsPerLane = 128 / ElSize;
 204   assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
 205          "Unexpected number of vector elements.");
 206
 207   for (unsigned i = 0; i != NumElts; ++i) {
 208     if (UndefElts[i]) {
 209       ShuffleMask.push_back(SM_SentinelUndef);
 210       continue;
 211     }
 212
 213     // VPERMIL2 Operation.
 214     // Bits[3] - Match Bit.
 215     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
 216     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
 217     uint64_t Selector = RawMask[i];
 218     unsigned MatchBit = (Selector >> 3) & 0x1;
 219
 220     // M2Z[0:1]     MatchBit
 221     //   0Xb           X        Source selected by Selector index.
 222     //   10b           0        Source selected by Selector index.
 223     //   10b           1        Zero.
 224     //   11b           0        Zero.
 225     //   11b           1        Source selected by Selector index.
 226     if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
 227       ShuffleMask.push_back(SM_SentinelZero);
 228       continue;
 229     }
 230
 231     int Index = i & ~(NumEltsPerLane - 1);
 232     if (ElSize == 64)
 233       Index += (Selector >> 1) & 0x1;
 234     else
 235       Index += Selector & 0x3;
 236
 237     int Src = (Selector >> 2) & 0x1;
 238     Index += Src * NumElts;
 239     ShuffleMask.push_back(Index);
 240   }
 241 }
 242
 243 void DecodeVPPERMMask(const Constant *C, unsigned Width,
 244                       SmallVectorImpl<int> &ShuffleMask) {
 245   Type *MaskTy = C->getType();
 246   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
 247   (void)MaskTySize;
 248   assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
 249
 250   // The shuffle mask requires a byte vector.
 251   APInt UndefElts;
 252   SmallVector<uint64_t, 16> RawMask;
 253   if (!extractConstantMask(C, 8, UndefElts, RawMask))
 254     return;
 255
 256   unsigned NumElts = Width / 8;
 257   assert(NumElts == 16 && "Unexpected number of vector elements.");
 258
 259   for (unsigned i = 0; i != NumElts; ++i) {
 260     if (UndefElts[i]) {
 261       ShuffleMask.push_back(SM_SentinelUndef);
 262       continue;
 263     }
 264
 265     // VPPERM Operation
 266     // Bits[4:0] - Byte Index (0 - 31)
 267     // Bits[7:5] - Permute Operation
 268     //
 269     // Permute Operation:
 270     // 0 - Source byte (no logical operation).
 271     // 1 - Invert source byte.
 272     // 2 - Bit reverse of source byte.
 273     // 3 - Bit reverse of inverted source byte.
 274     // 4 - 00h (zero - fill).
 275     // 5 - FFh (ones - fill).
 276     // 6 - Most significant bit of source byte replicated in all bit positions.
 277     // 7 - Invert most significant bit of source byte and replicate in all bit
 278     // positions.
 279     uint64_t Element = RawMask[i];
 280     uint64_t Index = Element & 0x1F;
 281     uint64_t PermuteOp = (Element >> 5) & 0x7;
 282
 283     if (PermuteOp == 4) {
 284       ShuffleMask.push_back(SM_SentinelZero);
 285       continue;
 286     }
 287     if (PermuteOp != 0) {
 288       ShuffleMask.clear();
 289       return;
 290     }
 291     ShuffleMask.push_back((int)Index);
 292   }
 293 }
 294
 295 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
 296                       SmallVectorImpl<int> &ShuffleMask) {
 297   assert((Width == 128 || Width == 256 || Width == 512) &&
 298          C->getType()->getPrimitiveSizeInBits() >= Width &&
 299          "Unexpected vector size.");
 300   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
 301          "Unexpected vector element size.");
 302
 303   // The shuffle mask requires elements the same size as the target.
 304   APInt UndefElts;
 305   SmallVector<uint64_t, 64> RawMask;
 306   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
 307     return;
 308
 309   unsigned NumElts = Width / ElSize;
 310
 311   for (unsigned i = 0; i != NumElts; ++i) {
 312     if (UndefElts[i]) {
 313       ShuffleMask.push_back(SM_SentinelUndef);
 314       continue;
 315     }
 316     int Index = RawMask[i] & (NumElts - 1);
 317     ShuffleMask.push_back(Index);
 318   }
 319 }
 320
 321 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
 322                        SmallVectorImpl<int> &ShuffleMask) {
 323   assert((Width == 128 || Width == 256 || Width == 512) &&
 324          C->getType()->getPrimitiveSizeInBits() >= Width &&
 325          "Unexpected vector size.");
 326   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
 327          "Unexpected vector element size.");
 328
 329   // The shuffle mask requires elements the same size as the target.
 330   APInt UndefElts;
 331   SmallVector<uint64_t, 64> RawMask;
 332   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
 333     return;
 334
 335   unsigned NumElts = Width / ElSize;
 336
 337   for (unsigned i = 0; i != NumElts; ++i) {
 338     if (UndefElts[i]) {
 339       ShuffleMask.push_back(SM_SentinelUndef);
 340       continue;
 341     }
 342     int Index = RawMask[i] & (NumElts*2 - 1);
 343     ShuffleMask.push_back(Index);
 344   }
 345 }
 346 } // llvm namespace