?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[InstCombine] Remove insertRangeTest code that handles the equality case.
[llvm-complete.git] / lib / CodeGen / SelectionDAG / LegalizeVectorTypes.cpp
blob7e4d52617977a4ea342b32f4b5c9c60b8b7a3e79
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include "llvm/Support/raw_ostream.h"
26 using namespace llvm;
27
28 #define DEBUG_TYPE "legalize-types"
29
30 //===----------------------------------------------------------------------===//
31 // Result Vector Scalarization: <1 x ty> -> ty.
32 //===----------------------------------------------------------------------===//
33
34 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
35 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
36 dbgs() << "\n");
37 SDValue R = SDValue();
38
39 switch (N->getOpcode()) {
40 default:
41 #ifndef NDEBUG
42 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
43 N->dump(&DAG);
44 dbgs() << "\n";
45 #endif
46 report_fatal_error("Do not know how to scalarize the result of this "
47 "operator!\n");
48
49 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
50 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
51 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
53 case ISD::STRICT_FP_ROUND: R = ScalarizeVecRes_STRICT_FP_ROUND(N); break;
54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
67 case ISD::ANY_EXTEND_VECTOR_INREG:
68 case ISD::SIGN_EXTEND_VECTOR_INREG:
69 case ISD::ZERO_EXTEND_VECTOR_INREG:
70 R = ScalarizeVecRes_VecInregOp(N);
71 break;
72 case ISD::ABS:
73 case ISD::ANY_EXTEND:
74 case ISD::BITREVERSE:
75 case ISD::BSWAP:
76 case ISD::CTLZ:
77 case ISD::CTLZ_ZERO_UNDEF:
78 case ISD::CTPOP:
79 case ISD::CTTZ:
80 case ISD::CTTZ_ZERO_UNDEF:
81 case ISD::FABS:
82 case ISD::FCEIL:
83 case ISD::FCOS:
84 case ISD::FEXP:
85 case ISD::FEXP2:
86 case ISD::FFLOOR:
87 case ISD::FLOG:
88 case ISD::FLOG10:
89 case ISD::FLOG2:
90 case ISD::FNEARBYINT:
91 case ISD::FNEG:
92 case ISD::FP_EXTEND:
93 case ISD::FP_TO_SINT:
94 case ISD::FP_TO_UINT:
95 case ISD::FRINT:
96 case ISD::FROUND:
97 case ISD::FSIN:
98 case ISD::FSQRT:
99 case ISD::FTRUNC:
100 case ISD::SIGN_EXTEND:
101 case ISD::SINT_TO_FP:
102 case ISD::TRUNCATE:
103 case ISD::UINT_TO_FP:
104 case ISD::ZERO_EXTEND:
105 case ISD::FCANONICALIZE:
106 R = ScalarizeVecRes_UnaryOp(N);
107 break;
108
109 case ISD::ADD:
110 case ISD::AND:
111 case ISD::FADD:
112 case ISD::FCOPYSIGN:
113 case ISD::FDIV:
114 case ISD::FMUL:
115 case ISD::FMINNUM:
116 case ISD::FMAXNUM:
117 case ISD::FMINNUM_IEEE:
118 case ISD::FMAXNUM_IEEE:
119 case ISD::FMINIMUM:
120 case ISD::FMAXIMUM:
121 case ISD::SMIN:
122 case ISD::SMAX:
123 case ISD::UMIN:
124 case ISD::UMAX:
125
126 case ISD::SADDSAT:
127 case ISD::UADDSAT:
128 case ISD::SSUBSAT:
129 case ISD::USUBSAT:
130
131 case ISD::FPOW:
132 case ISD::FREM:
133 case ISD::FSUB:
134 case ISD::MUL:
135 case ISD::OR:
136 case ISD::SDIV:
137 case ISD::SREM:
138 case ISD::SUB:
139 case ISD::UDIV:
140 case ISD::UREM:
141 case ISD::XOR:
142 case ISD::SHL:
143 case ISD::SRA:
144 case ISD::SRL:
145 R = ScalarizeVecRes_BinOp(N);
146 break;
147 case ISD::FMA:
148 R = ScalarizeVecRes_TernaryOp(N);
149 break;
150 case ISD::STRICT_FADD:
151 case ISD::STRICT_FSUB:
152 case ISD::STRICT_FMUL:
153 case ISD::STRICT_FDIV:
154 case ISD::STRICT_FREM:
155 case ISD::STRICT_FSQRT:
156 case ISD::STRICT_FMA:
157 case ISD::STRICT_FPOW:
158 case ISD::STRICT_FPOWI:
159 case ISD::STRICT_FSIN:
160 case ISD::STRICT_FCOS:
161 case ISD::STRICT_FEXP:
162 case ISD::STRICT_FEXP2:
163 case ISD::STRICT_FLOG:
164 case ISD::STRICT_FLOG10:
165 case ISD::STRICT_FLOG2:
166 case ISD::STRICT_FRINT:
167 case ISD::STRICT_FNEARBYINT:
168 case ISD::STRICT_FMAXNUM:
169 case ISD::STRICT_FMINNUM:
170 case ISD::STRICT_FCEIL:
171 case ISD::STRICT_FFLOOR:
172 case ISD::STRICT_FROUND:
173 case ISD::STRICT_FTRUNC:
174 case ISD::STRICT_FP_EXTEND:
175 R = ScalarizeVecRes_StrictFPOp(N);
176 break;
177 case ISD::UADDO:
178 case ISD::SADDO:
179 case ISD::USUBO:
180 case ISD::SSUBO:
181 case ISD::UMULO:
182 case ISD::SMULO:
183 R = ScalarizeVecRes_OverflowOp(N, ResNo);
184 break;
185 case ISD::SMULFIX:
186 case ISD::SMULFIXSAT:
187 case ISD::UMULFIX:
188 R = ScalarizeVecRes_MULFIX(N);
189 break;
190 }
191
192 // If R is null, the sub-method took care of registering the result.
193 if (R.getNode())
194 SetScalarizedVector(SDValue(N, ResNo), R);
195 }
196
197 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
198 SDValue LHS = GetScalarizedVector(N->getOperand(0));
199 SDValue RHS = GetScalarizedVector(N->getOperand(1));
200 return DAG.getNode(N->getOpcode(), SDLoc(N),
201 LHS.getValueType(), LHS, RHS, N->getFlags());
202 }
203
204 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
205 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
206 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
207 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
208 return DAG.getNode(N->getOpcode(), SDLoc(N),
209 Op0.getValueType(), Op0, Op1, Op2);
210 }
211
212 SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
213 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
214 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
215 SDValue Op2 = N->getOperand(2);
216 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
217 Op2);
218 }
219
220 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
221 EVT VT = N->getValueType(0).getVectorElementType();
222 unsigned NumOpers = N->getNumOperands();
223 SDValue Chain = N->getOperand(0);
224 EVT ValueVTs[] = {VT, MVT::Other};
225 SDLoc dl(N);
226
227 SmallVector<SDValue, 4> Opers;
228
229 // The Chain is the first operand.
230 Opers.push_back(Chain);
231
232 // Now process the remaining operands.
233 for (unsigned i = 1; i < NumOpers; ++i) {
234 SDValue Oper = N->getOperand(i);
235
236 if (Oper.getValueType().isVector())
237 Oper = GetScalarizedVector(Oper);
238
239 Opers.push_back(Oper);
240 }
241
242 SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
243
244 // Legalize the chain result - switch anything that used the old chain to
245 // use the new one.
246 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247 return Result;
248 }
249
250 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251 unsigned ResNo) {
252 SDLoc DL(N);
253 EVT ResVT = N->getValueType(0);
254 EVT OvVT = N->getValueType(1);
255
256 SDValue ScalarLHS, ScalarRHS;
257 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258 ScalarLHS = GetScalarizedVector(N->getOperand(0));
259 ScalarRHS = GetScalarizedVector(N->getOperand(1));
260 } else {
261 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264 ScalarLHS = ElemsLHS[0];
265 ScalarRHS = ElemsRHS[0];
266 }
267
268 SDVTList ScalarVTs = DAG.getVTList(
269 ResVT.getVectorElementType(), OvVT.getVectorElementType());
270 SDNode *ScalarNode = DAG.getNode(
271 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272
273 // Replace the other vector result not being explicitly scalarized here.
274 unsigned OtherNo = 1 - ResNo;
275 EVT OtherVT = N->getValueType(OtherNo);
276 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
277 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
278 } else {
279 SDValue OtherVal = DAG.getNode(
280 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
281 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
282 }
283
284 return SDValue(ScalarNode, ResNo);
285 }
286
287 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
288 unsigned ResNo) {
289 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
290 return GetScalarizedVector(Op);
291 }
292
293 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
294 SDValue Op = N->getOperand(0);
295 if (Op.getValueType().isVector()
296 && Op.getValueType().getVectorNumElements() == 1
297 && !isSimpleLegalType(Op.getValueType()))
298 Op = GetScalarizedVector(Op);
299 EVT NewVT = N->getValueType(0).getVectorElementType();
300 return DAG.getNode(ISD::BITCAST, SDLoc(N),
301 NewVT, Op);
302 }
303
304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
305 EVT EltVT = N->getValueType(0).getVectorElementType();
306 SDValue InOp = N->getOperand(0);
307 // The BUILD_VECTOR operands may be of wider element types and
308 // we may need to truncate them back to the requested return type.
309 if (EltVT.isInteger())
310 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
311 return InOp;
312 }
313
314 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
315 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
316 N->getValueType(0).getVectorElementType(),
317 N->getOperand(0), N->getOperand(1));
318 }
319
320 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
321 EVT NewVT = N->getValueType(0).getVectorElementType();
322 SDValue Op = GetScalarizedVector(N->getOperand(0));
323 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
324 NewVT, Op, N->getOperand(1));
325 }
326
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N) {
328 EVT NewVT = N->getValueType(0).getVectorElementType();
329 SDValue Op = GetScalarizedVector(N->getOperand(1));
330 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
331 { NewVT, MVT::Other },
332 { N->getOperand(0), Op, N->getOperand(2) });
333 // Legalize the chain result - switch anything that used the old chain to
334 // use the new one.
335 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
336 return Res;
337 }
338
339 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340 SDValue Op = GetScalarizedVector(N->getOperand(0));
341 return DAG.getNode(ISD::FPOWI, SDLoc(N),
342 Op.getValueType(), Op, N->getOperand(1));
343 }
344
345 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346 // The value to insert may have a wider type than the vector element type,
347 // so be sure to truncate it to the element type if necessary.
348 SDValue Op = N->getOperand(1);
349 EVT EltVT = N->getValueType(0).getVectorElementType();
350 if (Op.getValueType() != EltVT)
351 // FIXME: Can this happen for floating point types?
352 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353 return Op;
354 }
355
356 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357 assert(N->isUnindexed() && "Indexed vector load?");
358
359 SDValue Result = DAG.getLoad(
360 ISD::UNINDEXED, N->getExtensionType(),
361 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
362 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
363 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
364 N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
365 N->getAAInfo());
366
367 // Legalize the chain result - switch anything that used the old chain to
368 // use the new one.
369 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
370 return Result;
371 }
372
373 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
374 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
375 EVT DestVT = N->getValueType(0).getVectorElementType();
376 SDValue Op = N->getOperand(0);
377 EVT OpVT = Op.getValueType();
378 SDLoc DL(N);
379 // The result needs scalarizing, but it's not a given that the source does.
380 // This is a workaround for targets where it's impossible to scalarize the
381 // result of a conversion, because the source type is legal.
382 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
383 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
384 // legal and was not scalarized.
385 // See the similar logic in ScalarizeVecRes_SETCC
386 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
387 Op = GetScalarizedVector(Op);
388 } else {
389 EVT VT = OpVT.getVectorElementType();
390 Op = DAG.getNode(
391 ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
392 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
393 }
394 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
395 }
396
397 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
398 EVT EltVT = N->getValueType(0).getVectorElementType();
399 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
400 SDValue LHS = GetScalarizedVector(N->getOperand(0));
401 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
402 LHS, DAG.getValueType(ExtVT));
403 }
404
405 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
406 SDLoc DL(N);
407 SDValue Op = N->getOperand(0);
408
409 EVT OpVT = Op.getValueType();
410 EVT OpEltVT = OpVT.getVectorElementType();
411 EVT EltVT = N->getValueType(0).getVectorElementType();
412
413 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
414 Op = GetScalarizedVector(Op);
415 } else {
416 Op = DAG.getNode(
417 ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
418 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
419 }
420
421 switch (N->getOpcode()) {
422 case ISD::ANY_EXTEND_VECTOR_INREG:
423 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
424 case ISD::SIGN_EXTEND_VECTOR_INREG:
425 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
426 case ISD::ZERO_EXTEND_VECTOR_INREG:
427 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
428 }
429
430 llvm_unreachable("Illegal extend_vector_inreg opcode");
431 }
432
433 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
434 // If the operand is wider than the vector element type then it is implicitly
435 // truncated. Make that explicit here.
436 EVT EltVT = N->getValueType(0).getVectorElementType();
437 SDValue InOp = N->getOperand(0);
438 if (InOp.getValueType() != EltVT)
439 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
440 return InOp;
441 }
442
443 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
444 SDValue Cond = N->getOperand(0);
445 EVT OpVT = Cond.getValueType();
446 SDLoc DL(N);
447 // The vselect result and true/value operands needs scalarizing, but it's
448 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
449 // See the similar logic in ScalarizeVecRes_SETCC
450 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
451 Cond = GetScalarizedVector(Cond);
452 } else {
453 EVT VT = OpVT.getVectorElementType();
454 Cond = DAG.getNode(
455 ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
456 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
457 }
458
459 SDValue LHS = GetScalarizedVector(N->getOperand(1));
460 TargetLowering::BooleanContent ScalarBool =
461 TLI.getBooleanContents(false, false);
462 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
463
464 // If integer and float booleans have different contents then we can't
465 // reliably optimize in all cases. There is a full explanation for this in
466 // DAGCombiner::visitSELECT() where the same issue affects folding
467 // (select C, 0, 1) to (xor C, 1).
468 if (TLI.getBooleanContents(false, false) !=
469 TLI.getBooleanContents(false, true)) {
470 // At least try the common case where the boolean is generated by a
471 // comparison.
472 if (Cond->getOpcode() == ISD::SETCC) {
473 EVT OpVT = Cond->getOperand(0).getValueType();
474 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
475 VecBool = TLI.getBooleanContents(OpVT);
476 } else
477 ScalarBool = TargetLowering::UndefinedBooleanContent;
478 }
479
480 EVT CondVT = Cond.getValueType();
481 if (ScalarBool != VecBool) {
482 switch (ScalarBool) {
483 case TargetLowering::UndefinedBooleanContent:
484 break;
485 case TargetLowering::ZeroOrOneBooleanContent:
486 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
487 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
488 // Vector read from all ones, scalar expects a single 1 so mask.
489 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
490 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
491 break;
492 case TargetLowering::ZeroOrNegativeOneBooleanContent:
493 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
494 VecBool == TargetLowering::ZeroOrOneBooleanContent);
495 // Vector reads from a one, scalar from all ones so sign extend.
496 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
497 Cond, DAG.getValueType(MVT::i1));
498 break;
499 }
500 }
501
502 // Truncate the condition if needed
503 auto BoolVT = getSetCCResultType(CondVT);
504 if (BoolVT.bitsLT(CondVT))
505 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
506
507 return DAG.getSelect(SDLoc(N),
508 LHS.getValueType(), Cond, LHS,
509 GetScalarizedVector(N->getOperand(2)));
510 }
511
512 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
513 SDValue LHS = GetScalarizedVector(N->getOperand(1));
514 return DAG.getSelect(SDLoc(N),
515 LHS.getValueType(), N->getOperand(0), LHS,
516 GetScalarizedVector(N->getOperand(2)));
517 }
518
519 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
520 SDValue LHS = GetScalarizedVector(N->getOperand(2));
521 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
522 N->getOperand(0), N->getOperand(1),
523 LHS, GetScalarizedVector(N->getOperand(3)),
524 N->getOperand(4));
525 }
526
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
528 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
529 }
530
531 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
532 // Figure out if the scalar is the LHS or RHS and return it.
533 SDValue Arg = N->getOperand(2).getOperand(0);
534 if (Arg.isUndef())
535 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
536 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
537 return GetScalarizedVector(N->getOperand(Op));
538 }
539
540 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
541 assert(N->getValueType(0).isVector() &&
542 N->getOperand(0).getValueType().isVector() &&
543 "Operand types must be vectors");
544 SDValue LHS = N->getOperand(0);
545 SDValue RHS = N->getOperand(1);
546 EVT OpVT = LHS.getValueType();
547 EVT NVT = N->getValueType(0).getVectorElementType();
548 SDLoc DL(N);
549
550 // The result needs scalarizing, but it's not a given that the source does.
551 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
552 LHS = GetScalarizedVector(LHS);
553 RHS = GetScalarizedVector(RHS);
554 } else {
555 EVT VT = OpVT.getVectorElementType();
556 LHS = DAG.getNode(
557 ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
558 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
559 RHS = DAG.getNode(
560 ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
561 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
562 }
563
564 // Turn it into a scalar SETCC.
565 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
566 N->getOperand(2));
567 // Vectors may have a different boolean contents to scalars. Promote the
568 // value appropriately.
569 ISD::NodeType ExtendCode =
570 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
571 return DAG.getNode(ExtendCode, DL, NVT, Res);
572 }
573
574
575 //===----------------------------------------------------------------------===//
576 // Operand Vector Scalarization <1 x ty> -> ty.
577 //===----------------------------------------------------------------------===//
578
579 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
580 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
581 dbgs() << "\n");
582 SDValue Res = SDValue();
583
584 if (!Res.getNode()) {
585 switch (N->getOpcode()) {
586 default:
587 #ifndef NDEBUG
588 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
589 N->dump(&DAG);
590 dbgs() << "\n";
591 #endif
592 report_fatal_error("Do not know how to scalarize this operator's "
593 "operand!\n");
594 case ISD::BITCAST:
595 Res = ScalarizeVecOp_BITCAST(N);
596 break;
597 case ISD::ANY_EXTEND:
598 case ISD::ZERO_EXTEND:
599 case ISD::SIGN_EXTEND:
600 case ISD::TRUNCATE:
601 case ISD::FP_TO_SINT:
602 case ISD::FP_TO_UINT:
603 case ISD::SINT_TO_FP:
604 case ISD::UINT_TO_FP:
605 Res = ScalarizeVecOp_UnaryOp(N);
606 break;
607 case ISD::CONCAT_VECTORS:
608 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
609 break;
610 case ISD::EXTRACT_VECTOR_ELT:
611 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
612 break;
613 case ISD::VSELECT:
614 Res = ScalarizeVecOp_VSELECT(N);
615 break;
616 case ISD::SETCC:
617 Res = ScalarizeVecOp_VSETCC(N);
618 break;
619 case ISD::STORE:
620 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
621 break;
622 case ISD::STRICT_FP_ROUND:
623 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
624 break;
625 case ISD::FP_ROUND:
626 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
627 break;
628 case ISD::VECREDUCE_FADD:
629 case ISD::VECREDUCE_FMUL:
630 case ISD::VECREDUCE_ADD:
631 case ISD::VECREDUCE_MUL:
632 case ISD::VECREDUCE_AND:
633 case ISD::VECREDUCE_OR:
634 case ISD::VECREDUCE_XOR:
635 case ISD::VECREDUCE_SMAX:
636 case ISD::VECREDUCE_SMIN:
637 case ISD::VECREDUCE_UMAX:
638 case ISD::VECREDUCE_UMIN:
639 case ISD::VECREDUCE_FMAX:
640 case ISD::VECREDUCE_FMIN:
641 Res = ScalarizeVecOp_VECREDUCE(N);
642 break;
643 }
644 }
645
646 // If the result is null, the sub-method took care of registering results etc.
647 if (!Res.getNode()) return false;
648
649 // If the result is N, the sub-method updated N in place. Tell the legalizer
650 // core about this.
651 if (Res.getNode() == N)
652 return true;
653
654 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
655 "Invalid operand expansion");
656
657 ReplaceValueWith(SDValue(N, 0), Res);
658 return false;
659 }
660
661 /// If the value to convert is a vector that needs to be scalarized, it must be
662 /// <1 x ty>. Convert the element instead.
663 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
664 SDValue Elt = GetScalarizedVector(N->getOperand(0));
665 return DAG.getNode(ISD::BITCAST, SDLoc(N),
666 N->getValueType(0), Elt);
667 }
668
669 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
670 /// Do the operation on the element instead.
671 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
672 assert(N->getValueType(0).getVectorNumElements() == 1 &&
673 "Unexpected vector type!");
674 SDValue Elt = GetScalarizedVector(N->getOperand(0));
675 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
676 N->getValueType(0).getScalarType(), Elt);
677 // Revectorize the result so the types line up with what the uses of this
678 // expression expect.
679 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
680 }
681
682 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
683 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
684 SmallVector<SDValue, 8> Ops(N->getNumOperands());
685 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
686 Ops[i] = GetScalarizedVector(N->getOperand(i));
687 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
688 }
689
690 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
691 /// so just return the element, ignoring the index.
692 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
693 EVT VT = N->getValueType(0);
694 SDValue Res = GetScalarizedVector(N->getOperand(0));
695 if (Res.getValueType() != VT)
696 Res = VT.isFloatingPoint()
697 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
698 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
699 return Res;
700 }
701
702 /// If the input condition is a vector that needs to be scalarized, it must be
703 /// <1 x i1>, so just convert to a normal ISD::SELECT
704 /// (still with vector output type since that was acceptable if we got here).
705 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
706 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
707 EVT VT = N->getValueType(0);
708
709 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
710 N->getOperand(2));
711 }
712
713 /// If the operand is a vector that needs to be scalarized then the
714 /// result must be v1i1, so just convert to a scalar SETCC and wrap
715 /// with a scalar_to_vector since the res type is legal if we got here
716 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
717 assert(N->getValueType(0).isVector() &&
718 N->getOperand(0).getValueType().isVector() &&
719 "Operand types must be vectors");
720 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
721
722 EVT VT = N->getValueType(0);
723 SDValue LHS = GetScalarizedVector(N->getOperand(0));
724 SDValue RHS = GetScalarizedVector(N->getOperand(1));
725
726 EVT OpVT = N->getOperand(0).getValueType();
727 EVT NVT = VT.getVectorElementType();
728 SDLoc DL(N);
729 // Turn it into a scalar SETCC.
730 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
731 N->getOperand(2));
732
733 // Vectors may have a different boolean contents to scalars. Promote the
734 // value appropriately.
735 ISD::NodeType ExtendCode =
736 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
737
738 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
739
740 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
741 }
742
743 /// If the value to store is a vector that needs to be scalarized, it must be
744 /// <1 x ty>. Just store the element.
745 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
746 assert(N->isUnindexed() && "Indexed store of one-element vector?");
747 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
748 SDLoc dl(N);
749
750 if (N->isTruncatingStore())
751 return DAG.getTruncStore(
752 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
753 N->getBasePtr(), N->getPointerInfo(),
754 N->getMemoryVT().getVectorElementType(), N->getAlignment(),
755 N->getMemOperand()->getFlags(), N->getAAInfo());
756
757 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
758 N->getBasePtr(), N->getPointerInfo(),
759 N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
760 N->getAAInfo());
761 }
762
763 /// If the value to round is a vector that needs to be scalarized, it must be
764 /// <1 x ty>. Convert the element instead.
765 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
766 SDValue Elt = GetScalarizedVector(N->getOperand(0));
767 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
768 N->getValueType(0).getVectorElementType(), Elt,
769 N->getOperand(1));
770 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
771 }
772
773 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
774 unsigned OpNo) {
775 assert(OpNo == 1 && "Wrong operand for scalarization!");
776 SDValue Elt = GetScalarizedVector(N->getOperand(1));
777 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
778 { N->getValueType(0).getVectorElementType(),
779 MVT::Other },
780 { N->getOperand(0), Elt, N->getOperand(2) });
781 // Legalize the chain result - switch anything that used the old chain to
782 // use the new one.
783 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
784 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
785 }
786
787 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
788 SDValue Res = GetScalarizedVector(N->getOperand(0));
789 // Result type may be wider than element type.
790 if (Res.getValueType() != N->getValueType(0))
791 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
792 return Res;
793 }
794
795 //===----------------------------------------------------------------------===//
796 // Result Vector Splitting
797 //===----------------------------------------------------------------------===//
798
799 /// This method is called when the specified result of the specified node is
800 /// found to need vector splitting. At this point, the node may also have
801 /// invalid operands or may have other results that need legalization, we just
802 /// know that (at least) one result needs vector splitting.
803 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
804 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
805 SDValue Lo, Hi;
806
807 // See if the target wants to custom expand this node.
808 if (CustomLowerNode(N, N->getValueType(ResNo), true))
809 return;
810
811 switch (N->getOpcode()) {
812 default:
813 #ifndef NDEBUG
814 dbgs() << "SplitVectorResult #" << ResNo << ": ";
815 N->dump(&DAG);
816 dbgs() << "\n";
817 #endif
818 report_fatal_error("Do not know how to split the result of this "
819 "operator!\n");
820
821 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
822 case ISD::VSELECT:
823 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
824 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
825 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
826 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
827 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
828 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
829 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
830 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
831 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
832 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
833 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
834 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
835 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
836 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
837 case ISD::LOAD:
838 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
839 break;
840 case ISD::MLOAD:
841 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
842 break;
843 case ISD::MGATHER:
844 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
845 break;
846 case ISD::SETCC:
847 SplitVecRes_SETCC(N, Lo, Hi);
848 break;
849 case ISD::VECTOR_SHUFFLE:
850 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
851 break;
852 case ISD::VAARG:
853 SplitVecRes_VAARG(N, Lo, Hi);
854 break;
855
856 case ISD::ANY_EXTEND_VECTOR_INREG:
857 case ISD::SIGN_EXTEND_VECTOR_INREG:
858 case ISD::ZERO_EXTEND_VECTOR_INREG:
859 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
860 break;
861
862 case ISD::ABS:
863 case ISD::BITREVERSE:
864 case ISD::BSWAP:
865 case ISD::CTLZ:
866 case ISD::CTTZ:
867 case ISD::CTLZ_ZERO_UNDEF:
868 case ISD::CTTZ_ZERO_UNDEF:
869 case ISD::CTPOP:
870 case ISD::FABS:
871 case ISD::FCEIL:
872 case ISD::FCOS:
873 case ISD::FEXP:
874 case ISD::FEXP2:
875 case ISD::FFLOOR:
876 case ISD::FLOG:
877 case ISD::FLOG10:
878 case ISD::FLOG2:
879 case ISD::FNEARBYINT:
880 case ISD::FNEG:
881 case ISD::FP_EXTEND:
882 case ISD::STRICT_FP_EXTEND:
883 case ISD::FP_ROUND:
884 case ISD::STRICT_FP_ROUND:
885 case ISD::FP_TO_SINT:
886 case ISD::FP_TO_UINT:
887 case ISD::FRINT:
888 case ISD::FROUND:
889 case ISD::FSIN:
890 case ISD::FSQRT:
891 case ISD::FTRUNC:
892 case ISD::SINT_TO_FP:
893 case ISD::TRUNCATE:
894 case ISD::UINT_TO_FP:
895 case ISD::FCANONICALIZE:
896 SplitVecRes_UnaryOp(N, Lo, Hi);
897 break;
898
899 case ISD::ANY_EXTEND:
900 case ISD::SIGN_EXTEND:
901 case ISD::ZERO_EXTEND:
902 SplitVecRes_ExtendOp(N, Lo, Hi);
903 break;
904
905 case ISD::ADD:
906 case ISD::SUB:
907 case ISD::MUL:
908 case ISD::MULHS:
909 case ISD::MULHU:
910 case ISD::FADD:
911 case ISD::FSUB:
912 case ISD::FMUL:
913 case ISD::FMINNUM:
914 case ISD::FMAXNUM:
915 case ISD::FMINIMUM:
916 case ISD::FMAXIMUM:
917 case ISD::SDIV:
918 case ISD::UDIV:
919 case ISD::FDIV:
920 case ISD::FPOW:
921 case ISD::AND:
922 case ISD::OR:
923 case ISD::XOR:
924 case ISD::SHL:
925 case ISD::SRA:
926 case ISD::SRL:
927 case ISD::UREM:
928 case ISD::SREM:
929 case ISD::FREM:
930 case ISD::SMIN:
931 case ISD::SMAX:
932 case ISD::UMIN:
933 case ISD::UMAX:
934 case ISD::SADDSAT:
935 case ISD::UADDSAT:
936 case ISD::SSUBSAT:
937 case ISD::USUBSAT:
938 SplitVecRes_BinOp(N, Lo, Hi);
939 break;
940 case ISD::FMA:
941 SplitVecRes_TernaryOp(N, Lo, Hi);
942 break;
943 case ISD::STRICT_FADD:
944 case ISD::STRICT_FSUB:
945 case ISD::STRICT_FMUL:
946 case ISD::STRICT_FDIV:
947 case ISD::STRICT_FREM:
948 case ISD::STRICT_FSQRT:
949 case ISD::STRICT_FMA:
950 case ISD::STRICT_FPOW:
951 case ISD::STRICT_FPOWI:
952 case ISD::STRICT_FSIN:
953 case ISD::STRICT_FCOS:
954 case ISD::STRICT_FEXP:
955 case ISD::STRICT_FEXP2:
956 case ISD::STRICT_FLOG:
957 case ISD::STRICT_FLOG10:
958 case ISD::STRICT_FLOG2:
959 case ISD::STRICT_FRINT:
960 case ISD::STRICT_FNEARBYINT:
961 case ISD::STRICT_FMAXNUM:
962 case ISD::STRICT_FMINNUM:
963 case ISD::STRICT_FCEIL:
964 case ISD::STRICT_FFLOOR:
965 case ISD::STRICT_FROUND:
966 case ISD::STRICT_FTRUNC:
967 SplitVecRes_StrictFPOp(N, Lo, Hi);
968 break;
969 case ISD::UADDO:
970 case ISD::SADDO:
971 case ISD::USUBO:
972 case ISD::SSUBO:
973 case ISD::UMULO:
974 case ISD::SMULO:
975 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
976 break;
977 case ISD::SMULFIX:
978 case ISD::SMULFIXSAT:
979 case ISD::UMULFIX:
980 SplitVecRes_MULFIX(N, Lo, Hi);
981 break;
982 }
983
984 // If Lo/Hi is null, the sub-method took care of registering results etc.
985 if (Lo.getNode())
986 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
987 }
988
989 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
990 SDValue &Hi) {
991 SDValue LHSLo, LHSHi;
992 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
993 SDValue RHSLo, RHSHi;
994 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
995 SDLoc dl(N);
996
997 const SDNodeFlags Flags = N->getFlags();
998 unsigned Opcode = N->getOpcode();
999 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1000 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1001 }
1002
1003 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1004 SDValue &Hi) {
1005 SDValue Op0Lo, Op0Hi;
1006 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1007 SDValue Op1Lo, Op1Hi;
1008 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1009 SDValue Op2Lo, Op2Hi;
1010 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1011 SDLoc dl(N);
1012
1013 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
1014 Op0Lo, Op1Lo, Op2Lo);
1015 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
1016 Op0Hi, Op1Hi, Op2Hi);
1017 }
1018
1019 void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1020 SDValue LHSLo, LHSHi;
1021 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1022 SDValue RHSLo, RHSHi;
1023 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1024 SDLoc dl(N);
1025 SDValue Op2 = N->getOperand(2);
1026
1027 unsigned Opcode = N->getOpcode();
1028 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
1029 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
1030 }
1031
1032 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1033 SDValue &Hi) {
1034 // We know the result is a vector. The input may be either a vector or a
1035 // scalar value.
1036 EVT LoVT, HiVT;
1037 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1038 SDLoc dl(N);
1039
1040 SDValue InOp = N->getOperand(0);
1041 EVT InVT = InOp.getValueType();
1042
1043 // Handle some special cases efficiently.
1044 switch (getTypeAction(InVT)) {
1045 case TargetLowering::TypeLegal:
1046 case TargetLowering::TypePromoteInteger:
1047 case TargetLowering::TypePromoteFloat:
1048 case TargetLowering::TypeSoftenFloat:
1049 case TargetLowering::TypeScalarizeVector:
1050 case TargetLowering::TypeWidenVector:
1051 break;
1052 case TargetLowering::TypeExpandInteger:
1053 case TargetLowering::TypeExpandFloat:
1054 // A scalar to vector conversion, where the scalar needs expansion.
1055 // If the vector is being split in two then we can just convert the
1056 // expanded pieces.
1057 if (LoVT == HiVT) {
1058 GetExpandedOp(InOp, Lo, Hi);
1059 if (DAG.getDataLayout().isBigEndian())
1060 std::swap(Lo, Hi);
1061 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1062 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1063 return;
1064 }
1065 break;
1066 case TargetLowering::TypeSplitVector:
1067 // If the input is a vector that needs to be split, convert each split
1068 // piece of the input now.
1069 GetSplitVector(InOp, Lo, Hi);
1070 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1071 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1072 return;
1073 }
1074
1075 // In the general case, convert the input to an integer and split it by hand.
1076 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1077 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1078 if (DAG.getDataLayout().isBigEndian())
1079 std::swap(LoIntVT, HiIntVT);
1080
1081 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1082
1083 if (DAG.getDataLayout().isBigEndian())
1084 std::swap(Lo, Hi);
1085 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1086 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1087 }
1088
1089 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1090 SDValue &Hi) {
1091 EVT LoVT, HiVT;
1092 SDLoc dl(N);
1093 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1094 unsigned LoNumElts = LoVT.getVectorNumElements();
1095 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1096 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1097
1098 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1099 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1100 }
1101
1102 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1103 SDValue &Hi) {
1104 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1105 SDLoc dl(N);
1106 unsigned NumSubvectors = N->getNumOperands() / 2;
1107 if (NumSubvectors == 1) {
1108 Lo = N->getOperand(0);
1109 Hi = N->getOperand(1);
1110 return;
1111 }
1112
1113 EVT LoVT, HiVT;
1114 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1115
1116 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1117 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1118
1119 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1120 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1121 }
1122
1123 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1124 SDValue &Hi) {
1125 SDValue Vec = N->getOperand(0);
1126 SDValue Idx = N->getOperand(1);
1127 SDLoc dl(N);
1128
1129 EVT LoVT, HiVT;
1130 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1131
1132 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1133 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1134 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1135 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1136 TLI.getVectorIdxTy(DAG.getDataLayout())));
1137 }
1138
1139 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1140 SDValue &Hi) {
1141 SDValue Vec = N->getOperand(0);
1142 SDValue SubVec = N->getOperand(1);
1143 SDValue Idx = N->getOperand(2);
1144 SDLoc dl(N);
1145 GetSplitVector(Vec, Lo, Hi);
1146
1147 EVT VecVT = Vec.getValueType();
1148 unsigned VecElems = VecVT.getVectorNumElements();
1149 unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1150
1151 // If we know the index is 0, and we know the subvector doesn't cross the
1152 // boundary between the halves, we can avoid spilling the vector, and insert
1153 // into the lower half of the split vector directly.
1154 // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1155 // the index is constant and there is no boundary crossing. But those cases
1156 // don't seem to get hit in practice.
1157 if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
1158 unsigned IdxVal = ConstIdx->getZExtValue();
1159 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1160 EVT LoVT, HiVT;
1161 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1162 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1163 return;
1164 }
1165 }
1166
1167 // Spill the vector to the stack.
1168 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1169 SDValue Store =
1170 DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1171
1172 // Store the new subvector into the specified index.
1173 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1174 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1175 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1176 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1177
1178 // Load the Lo part from the stack slot.
1179 Lo =
1180 DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1181
1182 // Increment the pointer to the other part.
1183 unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1184 StackPtr =
1185 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1186 DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1187
1188 // Load the Hi part from the stack slot.
1189 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1190 MinAlign(Alignment, IncrementSize));
1191 }
1192
1193 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1194 SDValue &Hi) {
1195 SDLoc dl(N);
1196 GetSplitVector(N->getOperand(0), Lo, Hi);
1197 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1198 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1199 }
1200
1201 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1202 SDValue &Hi) {
1203 SDValue LHSLo, LHSHi;
1204 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1205 SDLoc DL(N);
1206
1207 SDValue RHSLo, RHSHi;
1208 SDValue RHS = N->getOperand(1);
1209 EVT RHSVT = RHS.getValueType();
1210 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1211 GetSplitVector(RHS, RHSLo, RHSHi);
1212 else
1213 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1214
1215
1216 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1217 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1218 }
1219
1220 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1221 SDValue &Hi) {
1222 SDValue LHSLo, LHSHi;
1223 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1224 SDLoc dl(N);
1225
1226 EVT LoVT, HiVT;
1227 std::tie(LoVT, HiVT) =
1228 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1229
1230 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1231 DAG.getValueType(LoVT));
1232 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1233 DAG.getValueType(HiVT));
1234 }
1235
1236 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1237 SDValue &Hi) {
1238 unsigned Opcode = N->getOpcode();
1239 SDValue N0 = N->getOperand(0);
1240
1241 SDLoc dl(N);
1242 SDValue InLo, InHi;
1243
1244 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1245 GetSplitVector(N0, InLo, InHi);
1246 else
1247 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1248
1249 EVT InLoVT = InLo.getValueType();
1250 unsigned InNumElements = InLoVT.getVectorNumElements();
1251
1252 EVT OutLoVT, OutHiVT;
1253 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1254 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1255 assert((2 * OutNumElements) <= InNumElements &&
1256 "Illegal extend vector in reg split");
1257
1258 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1259 // input vector (i.e. we only use InLo):
1260 // OutLo will extend the first OutNumElements from InLo.
1261 // OutHi will extend the next OutNumElements from InLo.
1262
1263 // Shuffle the elements from InLo for OutHi into the bottom elements to
1264 // create a 'fake' InHi.
1265 SmallVector<int, 8> SplitHi(InNumElements, -1);
1266 for (unsigned i = 0; i != OutNumElements; ++i)
1267 SplitHi[i] = i + OutNumElements;
1268 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1269
1270 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1271 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1272 }
1273
1274 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1275 SDValue &Hi) {
1276 unsigned NumOps = N->getNumOperands();
1277 SDValue Chain = N->getOperand(0);
1278 EVT LoVT, HiVT;
1279 SDLoc dl(N);
1280 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1281
1282 SmallVector<SDValue, 4> OpsLo;
1283 SmallVector<SDValue, 4> OpsHi;
1284
1285 // The Chain is the first operand.
1286 OpsLo.push_back(Chain);
1287 OpsHi.push_back(Chain);
1288
1289 // Now process the remaining operands.
1290 for (unsigned i = 1; i < NumOps; ++i) {
1291 SDValue Op = N->getOperand(i);
1292 SDValue OpLo = Op;
1293 SDValue OpHi = Op;
1294
1295 EVT InVT = Op.getValueType();
1296 if (InVT.isVector()) {
1297 // If the input also splits, handle it directly for a
1298 // compile time speedup. Otherwise split it by hand.
1299 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1300 GetSplitVector(Op, OpLo, OpHi);
1301 else
1302 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1303 }
1304
1305 OpsLo.push_back(OpLo);
1306 OpsHi.push_back(OpHi);
1307 }
1308
1309 EVT LoValueVTs[] = {LoVT, MVT::Other};
1310 EVT HiValueVTs[] = {HiVT, MVT::Other};
1311 Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1312 Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1313
1314 // Build a factor node to remember that this Op is independent of the
1315 // other one.
1316 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1317 Lo.getValue(1), Hi.getValue(1));
1318
1319 // Legalize the chain result - switch anything that used the old chain to
1320 // use the new one.
1321 ReplaceValueWith(SDValue(N, 1), Chain);
1322 }
1323
1324 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1325 SDValue Chain = N->getOperand(0);
1326 EVT VT = N->getValueType(0);
1327 unsigned NE = VT.getVectorNumElements();
1328 EVT EltVT = VT.getVectorElementType();
1329 SDLoc dl(N);
1330
1331 SmallVector<SDValue, 8> Scalars;
1332 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1333
1334 // If ResNE is 0, fully unroll the vector op.
1335 if (ResNE == 0)
1336 ResNE = NE;
1337 else if (NE > ResNE)
1338 NE = ResNE;
1339
1340 //The results of each unrolled operation, including the chain.
1341 EVT ChainVTs[] = {EltVT, MVT::Other};
1342 SmallVector<SDValue, 8> Chains;
1343
1344 unsigned i;
1345 for (i = 0; i != NE; ++i) {
1346 Operands[0] = Chain;
1347 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1348 SDValue Operand = N->getOperand(j);
1349 EVT OperandVT = Operand.getValueType();
1350 if (OperandVT.isVector()) {
1351 EVT OperandEltVT = OperandVT.getVectorElementType();
1352 Operands[j] =
1353 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
1354 DAG.getConstant(i, dl, TLI.getVectorIdxTy(
1355 DAG.getDataLayout())));
1356 } else {
1357 Operands[j] = Operand;
1358 }
1359 }
1360 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1361 Scalar.getNode()->setFlags(N->getFlags());
1362
1363 //Add in the scalar as well as its chain value to the
1364 //result vectors.
1365 Scalars.push_back(Scalar);
1366 Chains.push_back(Scalar.getValue(1));
1367 }
1368
1369 for (; i < ResNE; ++i)
1370 Scalars.push_back(DAG.getUNDEF(EltVT));
1371
1372 // Build a new factor node to connect the chain back together.
1373 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1374 ReplaceValueWith(SDValue(N, 1), Chain);
1375
1376 // Create a new BUILD_VECTOR node
1377 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1378 return DAG.getBuildVector(VecVT, dl, Scalars);
1379 }
1380
1381 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1382 SDValue &Lo, SDValue &Hi) {
1383 SDLoc dl(N);
1384 EVT ResVT = N->getValueType(0);
1385 EVT OvVT = N->getValueType(1);
1386 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1387 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1388 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1389
1390 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1391 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1392 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1393 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1394 } else {
1395 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1396 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1397 }
1398
1399 unsigned Opcode = N->getOpcode();
1400 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1401 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1402 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1403 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1404
1405 Lo = SDValue(LoNode, ResNo);
1406 Hi = SDValue(HiNode, ResNo);
1407
1408 // Replace the other vector result not being explicitly split here.
1409 unsigned OtherNo = 1 - ResNo;
1410 EVT OtherVT = N->getValueType(OtherNo);
1411 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1412 SetSplitVector(SDValue(N, OtherNo),
1413 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1414 } else {
1415 SDValue OtherVal = DAG.getNode(
1416 ISD::CONCAT_VECTORS, dl, OtherVT,
1417 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1418 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1419 }
1420 }
1421
1422 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1423 SDValue &Hi) {
1424 SDValue Vec = N->getOperand(0);
1425 SDValue Elt = N->getOperand(1);
1426 SDValue Idx = N->getOperand(2);
1427 SDLoc dl(N);
1428 GetSplitVector(Vec, Lo, Hi);
1429
1430 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1431 unsigned IdxVal = CIdx->getZExtValue();
1432 unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1433 if (IdxVal < LoNumElts)
1434 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1435 Lo.getValueType(), Lo, Elt, Idx);
1436 else
1437 Hi =
1438 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1439 DAG.getConstant(IdxVal - LoNumElts, dl,
1440 TLI.getVectorIdxTy(DAG.getDataLayout())));
1441 return;
1442 }
1443
1444 // See if the target wants to custom expand this node.
1445 if (CustomLowerNode(N, N->getValueType(0), true))
1446 return;
1447
1448 // Make the vector elements byte-addressable if they aren't already.
1449 EVT VecVT = Vec.getValueType();
1450 EVT EltVT = VecVT.getVectorElementType();
1451 if (VecVT.getScalarSizeInBits() < 8) {
1452 EltVT = MVT::i8;
1453 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1454 VecVT.getVectorNumElements());
1455 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1456 // Extend the element type to match if needed.
1457 if (EltVT.bitsGT(Elt.getValueType()))
1458 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1459 }
1460
1461 // Spill the vector to the stack.
1462 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1463 auto &MF = DAG.getMachineFunction();
1464 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1465 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1466 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1467
1468 // Store the new element. This may be larger than the vector element type,
1469 // so use a truncating store.
1470 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1471 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1472 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1473 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1474 MachinePointerInfo::getUnknownStack(MF), EltVT);
1475
1476 EVT LoVT, HiVT;
1477 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1478
1479 // Load the Lo part from the stack slot.
1480 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1481
1482 // Increment the pointer to the other part.
1483 unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1484 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1485 DAG.getConstant(IncrementSize, dl,
1486 StackPtr.getValueType()));
1487
1488 // Load the Hi part from the stack slot.
1489 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1490 PtrInfo.getWithOffset(IncrementSize),
1491 MinAlign(Alignment, IncrementSize));
1492
1493 // If we adjusted the original type, we need to truncate the results.
1494 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1495 if (LoVT != Lo.getValueType())
1496 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1497 if (HiVT != Hi.getValueType())
1498 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1499 }
1500
1501 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1502 SDValue &Hi) {
1503 EVT LoVT, HiVT;
1504 SDLoc dl(N);
1505 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1506 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1507 Hi = DAG.getUNDEF(HiVT);
1508 }
1509
1510 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1511 SDValue &Hi) {
1512 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1513 EVT LoVT, HiVT;
1514 SDLoc dl(LD);
1515 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1516
1517 ISD::LoadExtType ExtType = LD->getExtensionType();
1518 SDValue Ch = LD->getChain();
1519 SDValue Ptr = LD->getBasePtr();
1520 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1521 EVT MemoryVT = LD->getMemoryVT();
1522 unsigned Alignment = LD->getOriginalAlignment();
1523 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1524 AAMDNodes AAInfo = LD->getAAInfo();
1525
1526 EVT LoMemVT, HiMemVT;
1527 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1528
1529 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1530 LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1531
1532 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1533 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1534 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1535 LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1536 Alignment, MMOFlags, AAInfo);
1537
1538 // Build a factor node to remember that this load is independent of the
1539 // other one.
1540 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1541 Hi.getValue(1));
1542
1543 // Legalize the chain result - switch anything that used the old chain to
1544 // use the new one.
1545 ReplaceValueWith(SDValue(LD, 1), Ch);
1546 }
1547
1548 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1549 SDValue &Lo, SDValue &Hi) {
1550 EVT LoVT, HiVT;
1551 SDLoc dl(MLD);
1552 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1553
1554 SDValue Ch = MLD->getChain();
1555 SDValue Ptr = MLD->getBasePtr();
1556 SDValue Mask = MLD->getMask();
1557 SDValue PassThru = MLD->getPassThru();
1558 unsigned Alignment = MLD->getOriginalAlignment();
1559 ISD::LoadExtType ExtType = MLD->getExtensionType();
1560
1561 // Split Mask operand
1562 SDValue MaskLo, MaskHi;
1563 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1564 GetSplitVector(Mask, MaskLo, MaskHi);
1565 else
1566 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1567
1568 EVT MemoryVT = MLD->getMemoryVT();
1569 EVT LoMemVT, HiMemVT;
1570 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1571
1572 SDValue PassThruLo, PassThruHi;
1573 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1574 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1575 else
1576 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1577
1578 MachineMemOperand *MMO = DAG.getMachineFunction().
1579 getMachineMemOperand(MLD->getPointerInfo(),
1580 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1581 Alignment, MLD->getAAInfo(), MLD->getRanges());
1582
1583 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1584 ExtType, MLD->isExpandingLoad());
1585
1586 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1587 MLD->isExpandingLoad());
1588 unsigned HiOffset = LoMemVT.getStoreSize();
1589
1590 MMO = DAG.getMachineFunction().getMachineMemOperand(
1591 MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
1592 HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(),
1593 MLD->getRanges());
1594
1595 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1596 ExtType, MLD->isExpandingLoad());
1597
1598 // Build a factor node to remember that this load is independent of the
1599 // other one.
1600 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1601 Hi.getValue(1));
1602
1603 // Legalize the chain result - switch anything that used the old chain to
1604 // use the new one.
1605 ReplaceValueWith(SDValue(MLD, 1), Ch);
1606
1607 }
1608
1609 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1610 SDValue &Lo, SDValue &Hi) {
1611 EVT LoVT, HiVT;
1612 SDLoc dl(MGT);
1613 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1614
1615 SDValue Ch = MGT->getChain();
1616 SDValue Ptr = MGT->getBasePtr();
1617 SDValue Mask = MGT->getMask();
1618 SDValue PassThru = MGT->getPassThru();
1619 SDValue Index = MGT->getIndex();
1620 SDValue Scale = MGT->getScale();
1621 unsigned Alignment = MGT->getOriginalAlignment();
1622
1623 // Split Mask operand
1624 SDValue MaskLo, MaskHi;
1625 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1626 GetSplitVector(Mask, MaskLo, MaskHi);
1627 else
1628 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1629
1630 EVT MemoryVT = MGT->getMemoryVT();
1631 EVT LoMemVT, HiMemVT;
1632 // Split MemoryVT
1633 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1634
1635 SDValue PassThruLo, PassThruHi;
1636 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1637 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1638 else
1639 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1640
1641 SDValue IndexHi, IndexLo;
1642 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1643 GetSplitVector(Index, IndexLo, IndexHi);
1644 else
1645 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1646
1647 MachineMemOperand *MMO = DAG.getMachineFunction().
1648 getMachineMemOperand(MGT->getPointerInfo(),
1649 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1650 Alignment, MGT->getAAInfo(), MGT->getRanges());
1651
1652 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1653 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1654 MMO);
1655
1656 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1657 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1658 MMO);
1659
1660 // Build a factor node to remember that this load is independent of the
1661 // other one.
1662 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1663 Hi.getValue(1));
1664
1665 // Legalize the chain result - switch anything that used the old chain to
1666 // use the new one.
1667 ReplaceValueWith(SDValue(MGT, 1), Ch);
1668 }
1669
1670
1671 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1672 assert(N->getValueType(0).isVector() &&
1673 N->getOperand(0).getValueType().isVector() &&
1674 "Operand types must be vectors");
1675
1676 EVT LoVT, HiVT;
1677 SDLoc DL(N);
1678 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1679
1680 // If the input also splits, handle it directly. Otherwise split it by hand.
1681 SDValue LL, LH, RL, RH;
1682 if (getTypeAction(N->getOperand(0).getValueType()) ==
1683 TargetLowering::TypeSplitVector)
1684 GetSplitVector(N->getOperand(0), LL, LH);
1685 else
1686 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1687
1688 if (getTypeAction(N->getOperand(1).getValueType()) ==
1689 TargetLowering::TypeSplitVector)
1690 GetSplitVector(N->getOperand(1), RL, RH);
1691 else
1692 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1693
1694 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1695 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1696 }
1697
1698 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1699 SDValue &Hi) {
1700 // Get the dest types - they may not match the input types, e.g. int_to_fp.
1701 EVT LoVT, HiVT;
1702 SDLoc dl(N);
1703 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1704
1705 // If the input also splits, handle it directly for a compile time speedup.
1706 // Otherwise split it by hand.
1707 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1708 EVT InVT = N->getOperand(OpNo).getValueType();
1709 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1710 GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1711 else
1712 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1713
1714 if (N->getOpcode() == ISD::FP_ROUND) {
1715 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1716 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1717 } else if (N->getOpcode() == ISD::STRICT_FP_ROUND) {
1718 Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1719 { N->getOperand(0), Lo, N->getOperand(2) });
1720 Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1721 { N->getOperand(0), Hi, N->getOperand(2) });
1722 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1723 Lo.getValue(1), Hi.getValue(1));
1724 ReplaceValueWith(SDValue(N, 1), NewChain);
1725 } else if (N->isStrictFPOpcode()) {
1726 Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1727 { N->getOperand(0), Lo });
1728 Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1729 { N->getOperand(0), Hi });
1730 // Legalize the chain result - switch anything that used the old chain to
1731 // use the new one.
1732 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1733 Lo.getValue(1), Hi.getValue(1));
1734 ReplaceValueWith(SDValue(N, 1), NewChain);
1735 } else {
1736 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1737 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1738 }
1739 }
1740
1741 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1743 SDLoc dl(N);
1744 EVT SrcVT = N->getOperand(0).getValueType();
1745 EVT DestVT = N->getValueType(0);
1746 EVT LoVT, HiVT;
1747 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1748
1749 // We can do better than a generic split operation if the extend is doing
1750 // more than just doubling the width of the elements and the following are
1751 // true:
1752 // - The number of vector elements is even,
1753 // - the source type is legal,
1754 // - the type of a split source is illegal,
1755 // - the type of an extended (by doubling element size) source is legal, and
1756 // - the type of that extended source when split is legal.
1757 //
1758 // This won't necessarily completely legalize the operation, but it will
1759 // more effectively move in the right direction and prevent falling down
1760 // to scalarization in many cases due to the input vector being split too
1761 // far.
1762 unsigned NumElements = SrcVT.getVectorNumElements();
1763 if ((NumElements & 1) == 0 &&
1764 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1765 LLVMContext &Ctx = *DAG.getContext();
1766 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1767 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1768
1769 EVT SplitLoVT, SplitHiVT;
1770 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1771 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1772 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1773 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1774 N->dump(&DAG); dbgs() << "\n");
1775 // Extend the source vector by one step.
1776 SDValue NewSrc =
1777 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1778 // Get the low and high halves of the new, extended one step, vector.
1779 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1780 // Extend those vector halves the rest of the way.
1781 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1782 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1783 return;
1784 }
1785 }
1786 // Fall back to the generic unary operator splitting otherwise.
1787 SplitVecRes_UnaryOp(N, Lo, Hi);
1788 }
1789
1790 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1791 SDValue &Lo, SDValue &Hi) {
1792 // The low and high parts of the original input give four input vectors.
1793 SDValue Inputs[4];
1794 SDLoc dl(N);
1795 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1796 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1797 EVT NewVT = Inputs[0].getValueType();
1798 unsigned NewElts = NewVT.getVectorNumElements();
1799
1800 // If Lo or Hi uses elements from at most two of the four input vectors, then
1801 // express it as a vector shuffle of those two inputs. Otherwise extract the
1802 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1803 SmallVector<int, 16> Ops;
1804 for (unsigned High = 0; High < 2; ++High) {
1805 SDValue &Output = High ? Hi : Lo;
1806
1807 // Build a shuffle mask for the output, discovering on the fly which
1808 // input vectors to use as shuffle operands (recorded in InputUsed).
1809 // If building a suitable shuffle vector proves too hard, then bail
1810 // out with useBuildVector set.
1811 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1812 unsigned FirstMaskIdx = High * NewElts;
1813 bool useBuildVector = false;
1814 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1815 // The mask element. This indexes into the input.
1816 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1817
1818 // The input vector this mask element indexes into.
1819 unsigned Input = (unsigned)Idx / NewElts;
1820
1821 if (Input >= array_lengthof(Inputs)) {
1822 // The mask element does not index into any input vector.
1823 Ops.push_back(-1);
1824 continue;
1825 }
1826
1827 // Turn the index into an offset from the start of the input vector.
1828 Idx -= Input * NewElts;
1829
1830 // Find or create a shuffle vector operand to hold this input.
1831 unsigned OpNo;
1832 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1833 if (InputUsed[OpNo] == Input) {
1834 // This input vector is already an operand.
1835 break;
1836 } else if (InputUsed[OpNo] == -1U) {
1837 // Create a new operand for this input vector.
1838 InputUsed[OpNo] = Input;
1839 break;
1840 }
1841 }
1842
1843 if (OpNo >= array_lengthof(InputUsed)) {
1844 // More than two input vectors used! Give up on trying to create a
1845 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1846 useBuildVector = true;
1847 break;
1848 }
1849
1850 // Add the mask index for the new shuffle vector.
1851 Ops.push_back(Idx + OpNo * NewElts);
1852 }
1853
1854 if (useBuildVector) {
1855 EVT EltVT = NewVT.getVectorElementType();
1856 SmallVector<SDValue, 16> SVOps;
1857
1858 // Extract the input elements by hand.
1859 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1860 // The mask element. This indexes into the input.
1861 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1862
1863 // The input vector this mask element indexes into.
1864 unsigned Input = (unsigned)Idx / NewElts;
1865
1866 if (Input >= array_lengthof(Inputs)) {
1867 // The mask element is "undef" or indexes off the end of the input.
1868 SVOps.push_back(DAG.getUNDEF(EltVT));
1869 continue;
1870 }
1871
1872 // Turn the index into an offset from the start of the input vector.
1873 Idx -= Input * NewElts;
1874
1875 // Extract the vector element by hand.
1876 SVOps.push_back(DAG.getNode(
1877 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1878 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1879 }
1880
1881 // Construct the Lo/Hi output using a BUILD_VECTOR.
1882 Output = DAG.getBuildVector(NewVT, dl, SVOps);
1883 } else if (InputUsed[0] == -1U) {
1884 // No input vectors were used! The result is undefined.
1885 Output = DAG.getUNDEF(NewVT);
1886 } else {
1887 SDValue Op0 = Inputs[InputUsed[0]];
1888 // If only one input was used, use an undefined vector for the other.
1889 SDValue Op1 = InputUsed[1] == -1U ?
1890 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1891 // At least one input vector was used. Create a new shuffle vector.
1892 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1893 }
1894
1895 Ops.clear();
1896 }
1897 }
1898
1899 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1900 EVT OVT = N->getValueType(0);
1901 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
1902 SDValue Chain = N->getOperand(0);
1903 SDValue Ptr = N->getOperand(1);
1904 SDValue SV = N->getOperand(2);
1905 SDLoc dl(N);
1906
1907 const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment(
1908 NVT.getTypeForEVT(*DAG.getContext()));
1909
1910 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment);
1911 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment);
1912 Chain = Hi.getValue(1);
1913
1914 // Modified the chain - switch anything that used the old chain to use
1915 // the new one.
1916 ReplaceValueWith(SDValue(N, 1), Chain);
1917 }
1918
1919
1920 //===----------------------------------------------------------------------===//
1921 // Operand Vector Splitting
1922 //===----------------------------------------------------------------------===//
1923
1924 /// This method is called when the specified operand of the specified node is
1925 /// found to need vector splitting. At this point, all of the result types of
1926 /// the node are known to be legal, but other operands of the node may need
1927 /// legalization as well as the specified one.
1928 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1929 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1930 SDValue Res = SDValue();
1931
1932 // See if the target wants to custom split this node.
1933 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1934 return false;
1935
1936 if (!Res.getNode()) {
1937 switch (N->getOpcode()) {
1938 default:
1939 #ifndef NDEBUG
1940 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1941 N->dump(&DAG);
1942 dbgs() << "\n";
1943 #endif
1944 report_fatal_error("Do not know how to split this operator's "
1945 "operand!\n");
1946
1947 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1948 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1949 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1950 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1951 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1952 case ISD::TRUNCATE:
1953 Res = SplitVecOp_TruncateHelper(N);
1954 break;
1955 case ISD::STRICT_FP_ROUND:
1956 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1957 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1958 case ISD::STORE:
1959 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1960 break;
1961 case ISD::MSTORE:
1962 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1963 break;
1964 case ISD::MSCATTER:
1965 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1966 break;
1967 case ISD::MGATHER:
1968 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1969 break;
1970 case ISD::VSELECT:
1971 Res = SplitVecOp_VSELECT(N, OpNo);
1972 break;
1973 case ISD::SINT_TO_FP:
1974 case ISD::UINT_TO_FP:
1975 if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1976 Res = SplitVecOp_TruncateHelper(N);
1977 else
1978 Res = SplitVecOp_UnaryOp(N);
1979 break;
1980 case ISD::FP_TO_SINT:
1981 case ISD::FP_TO_UINT:
1982 case ISD::CTTZ:
1983 case ISD::CTLZ:
1984 case ISD::CTPOP:
1985 case ISD::STRICT_FP_EXTEND:
1986 case ISD::FP_EXTEND:
1987 case ISD::SIGN_EXTEND:
1988 case ISD::ZERO_EXTEND:
1989 case ISD::ANY_EXTEND:
1990 case ISD::FTRUNC:
1991 case ISD::FCANONICALIZE:
1992 Res = SplitVecOp_UnaryOp(N);
1993 break;
1994
1995 case ISD::ANY_EXTEND_VECTOR_INREG:
1996 case ISD::SIGN_EXTEND_VECTOR_INREG:
1997 case ISD::ZERO_EXTEND_VECTOR_INREG:
1998 Res = SplitVecOp_ExtVecInRegOp(N);
1999 break;
2000
2001 case ISD::VECREDUCE_FADD:
2002 case ISD::VECREDUCE_FMUL:
2003 case ISD::VECREDUCE_ADD:
2004 case ISD::VECREDUCE_MUL:
2005 case ISD::VECREDUCE_AND:
2006 case ISD::VECREDUCE_OR:
2007 case ISD::VECREDUCE_XOR:
2008 case ISD::VECREDUCE_SMAX:
2009 case ISD::VECREDUCE_SMIN:
2010 case ISD::VECREDUCE_UMAX:
2011 case ISD::VECREDUCE_UMIN:
2012 case ISD::VECREDUCE_FMAX:
2013 case ISD::VECREDUCE_FMIN:
2014 Res = SplitVecOp_VECREDUCE(N, OpNo);
2015 break;
2016 }
2017 }
2018
2019 // If the result is null, the sub-method took care of registering results etc.
2020 if (!Res.getNode()) return false;
2021
2022 // If the result is N, the sub-method updated N in place. Tell the legalizer
2023 // core about this.
2024 if (Res.getNode() == N)
2025 return true;
2026
2027 if (N->isStrictFPOpcode())
2028 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2029 "Invalid operand expansion");
2030 else
2031 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2032 "Invalid operand expansion");
2033
2034 ReplaceValueWith(SDValue(N, 0), Res);
2035 return false;
2036 }
2037
2038 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2039 // The only possibility for an illegal operand is the mask, since result type
2040 // legalization would have handled this node already otherwise.
2041 assert(OpNo == 0 && "Illegal operand must be mask");
2042
2043 SDValue Mask = N->getOperand(0);
2044 SDValue Src0 = N->getOperand(1);
2045 SDValue Src1 = N->getOperand(2);
2046 EVT Src0VT = Src0.getValueType();
2047 SDLoc DL(N);
2048 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2049
2050 SDValue Lo, Hi;
2051 GetSplitVector(N->getOperand(0), Lo, Hi);
2052 assert(Lo.getValueType() == Hi.getValueType() &&
2053 "Lo and Hi have differing types");
2054
2055 EVT LoOpVT, HiOpVT;
2056 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2057 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2058
2059 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2060 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2061 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2062 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2063
2064 SDValue LoSelect =
2065 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2066 SDValue HiSelect =
2067 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2068
2069 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2070 }
2071
2072 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2073 EVT ResVT = N->getValueType(0);
2074 SDValue Lo, Hi;
2075 SDLoc dl(N);
2076
2077 SDValue VecOp = N->getOperand(OpNo);
2078 EVT VecVT = VecOp.getValueType();
2079 assert(VecVT.isVector() && "Can only split reduce vector operand");
2080 GetSplitVector(VecOp, Lo, Hi);
2081 EVT LoOpVT, HiOpVT;
2082 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2083
2084 bool NoNaN = N->getFlags().hasNoNaNs();
2085 unsigned CombineOpc = 0;
2086 switch (N->getOpcode()) {
2087 case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
2088 case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
2089 case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
2090 case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
2091 case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
2092 case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
2093 case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
2094 case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
2095 case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
2096 case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
2097 case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
2098 case ISD::VECREDUCE_FMAX:
2099 CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM;
2100 break;
2101 case ISD::VECREDUCE_FMIN:
2102 CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
2103 break;
2104 default:
2105 llvm_unreachable("Unexpected reduce ISD node");
2106 }
2107
2108 // Use the appropriate scalar instruction on the split subvectors before
2109 // reducing the now partially reduced smaller vector.
2110 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2111 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2112 }
2113
2114 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2115 // The result has a legal vector type, but the input needs splitting.
2116 EVT ResVT = N->getValueType(0);
2117 SDValue Lo, Hi;
2118 SDLoc dl(N);
2119 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2120 EVT InVT = Lo.getValueType();
2121
2122 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2123 InVT.getVectorNumElements());
2124
2125 if (N->isStrictFPOpcode()) {
2126 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2127 { N->getOperand(0), Lo });
2128 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2129 { N->getOperand(0), Hi });
2130
2131 // Build a factor node to remember that this operation is independent
2132 // of the other one.
2133 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2134 Hi.getValue(1));
2135
2136 // Legalize the chain result - switch anything that used the old chain to
2137 // use the new one.
2138 ReplaceValueWith(SDValue(N, 1), Ch);
2139 } else {
2140 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2141 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2142 }
2143
2144 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2145 }
2146
2147 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2148 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2149 // end up being split all the way down to individual components. Convert the
2150 // split pieces into integers and reassemble.
2151 SDValue Lo, Hi;
2152 GetSplitVector(N->getOperand(0), Lo, Hi);
2153 Lo = BitConvertToInteger(Lo);
2154 Hi = BitConvertToInteger(Hi);
2155
2156 if (DAG.getDataLayout().isBigEndian())
2157 std::swap(Lo, Hi);
2158
2159 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2160 JoinIntegers(Lo, Hi));
2161 }
2162
2163 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2164 // We know that the extracted result type is legal.
2165 EVT SubVT = N->getValueType(0);
2166 SDValue Idx = N->getOperand(1);
2167 SDLoc dl(N);
2168 SDValue Lo, Hi;
2169 GetSplitVector(N->getOperand(0), Lo, Hi);
2170
2171 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2172 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2173
2174 if (IdxVal < LoElts) {
2175 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
2176 "Extracted subvector crosses vector split!");
2177 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2178 } else {
2179 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2180 DAG.getConstant(IdxVal - LoElts, dl,
2181 Idx.getValueType()));
2182 }
2183 }
2184
2185 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2186 SDValue Vec = N->getOperand(0);
2187 SDValue Idx = N->getOperand(1);
2188 EVT VecVT = Vec.getValueType();
2189
2190 if (isa<ConstantSDNode>(Idx)) {
2191 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2192
2193 SDValue Lo, Hi;
2194 GetSplitVector(Vec, Lo, Hi);
2195
2196 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2197
2198 if (IdxVal < LoElts)
2199 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2200 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2201 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2202 Idx.getValueType())), 0);
2203 }
2204
2205 // See if the target wants to custom expand this node.
2206 if (CustomLowerNode(N, N->getValueType(0), true))
2207 return SDValue();
2208
2209 // Make the vector elements byte-addressable if they aren't already.
2210 SDLoc dl(N);
2211 EVT EltVT = VecVT.getVectorElementType();
2212 if (VecVT.getScalarSizeInBits() < 8) {
2213 EltVT = MVT::i8;
2214 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2215 VecVT.getVectorNumElements());
2216 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2217 }
2218
2219 // Store the vector to the stack.
2220 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2221 auto &MF = DAG.getMachineFunction();
2222 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2223 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2224 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2225
2226 // Load back the required element.
2227 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2228
2229 // FIXME: This is to handle i1 vectors with elements promoted to i8.
2230 // i1 vector handling needs general improvement.
2231 if (N->getValueType(0).bitsLT(EltVT)) {
2232 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2233 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2234 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2235 }
2236
2237 return DAG.getExtLoad(
2238 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2239 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
2240 }
2241
2242 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2243 SDValue Lo, Hi;
2244
2245 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2246 // splitting the result has the same effect as splitting the input operand.
2247 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2248
2249 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2250 }
2251
2252 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2253 unsigned OpNo) {
2254 EVT LoVT, HiVT;
2255 SDLoc dl(MGT);
2256 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2257
2258 SDValue Ch = MGT->getChain();
2259 SDValue Ptr = MGT->getBasePtr();
2260 SDValue Index = MGT->getIndex();
2261 SDValue Scale = MGT->getScale();
2262 SDValue Mask = MGT->getMask();
2263 SDValue PassThru = MGT->getPassThru();
2264 unsigned Alignment = MGT->getOriginalAlignment();
2265
2266 SDValue MaskLo, MaskHi;
2267 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2268 // Split Mask operand
2269 GetSplitVector(Mask, MaskLo, MaskHi);
2270 else
2271 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2272
2273 EVT MemoryVT = MGT->getMemoryVT();
2274 EVT LoMemVT, HiMemVT;
2275 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2276
2277 SDValue PassThruLo, PassThruHi;
2278 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2279 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2280 else
2281 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2282
2283 SDValue IndexHi, IndexLo;
2284 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2285 GetSplitVector(Index, IndexLo, IndexHi);
2286 else
2287 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2288
2289 MachineMemOperand *MMO = DAG.getMachineFunction().
2290 getMachineMemOperand(MGT->getPointerInfo(),
2291 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
2292 Alignment, MGT->getAAInfo(), MGT->getRanges());
2293
2294 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2295 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2296 OpsLo, MMO);
2297
2298 MMO = DAG.getMachineFunction().
2299 getMachineMemOperand(MGT->getPointerInfo(),
2300 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
2301 Alignment, MGT->getAAInfo(),
2302 MGT->getRanges());
2303
2304 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2305 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2306 OpsHi, MMO);
2307
2308 // Build a factor node to remember that this load is independent of the
2309 // other one.
2310 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2311 Hi.getValue(1));
2312
2313 // Legalize the chain result - switch anything that used the old chain to
2314 // use the new one.
2315 ReplaceValueWith(SDValue(MGT, 1), Ch);
2316
2317 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2318 Hi);
2319 ReplaceValueWith(SDValue(MGT, 0), Res);
2320 return SDValue();
2321 }
2322
2323 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2324 unsigned OpNo) {
2325 SDValue Ch = N->getChain();
2326 SDValue Ptr = N->getBasePtr();
2327 SDValue Mask = N->getMask();
2328 SDValue Data = N->getValue();
2329 EVT MemoryVT = N->getMemoryVT();
2330 unsigned Alignment = N->getOriginalAlignment();
2331 SDLoc DL(N);
2332
2333 EVT LoMemVT, HiMemVT;
2334 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2335
2336 SDValue DataLo, DataHi;
2337 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2338 // Split Data operand
2339 GetSplitVector(Data, DataLo, DataHi);
2340 else
2341 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2342
2343 SDValue MaskLo, MaskHi;
2344 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2345 // Split Mask operand
2346 GetSplitVector(Mask, MaskLo, MaskHi);
2347 else
2348 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2349
2350 SDValue Lo, Hi;
2351 MachineMemOperand *MMO = DAG.getMachineFunction().
2352 getMachineMemOperand(N->getPointerInfo(),
2353 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2354 Alignment, N->getAAInfo(), N->getRanges());
2355
2356 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2357 N->isTruncatingStore(),
2358 N->isCompressingStore());
2359
2360 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2361 N->isCompressingStore());
2362 unsigned HiOffset = LoMemVT.getStoreSize();
2363
2364 MMO = DAG.getMachineFunction().getMachineMemOperand(
2365 N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
2366 HiMemVT.getStoreSize(), Alignment, N->getAAInfo(),
2367 N->getRanges());
2368
2369 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2370 N->isTruncatingStore(), N->isCompressingStore());
2371
2372 // Build a factor node to remember that this store is independent of the
2373 // other one.
2374 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2375 }
2376
2377 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2378 unsigned OpNo) {
2379 SDValue Ch = N->getChain();
2380 SDValue Ptr = N->getBasePtr();
2381 SDValue Mask = N->getMask();
2382 SDValue Index = N->getIndex();
2383 SDValue Scale = N->getScale();
2384 SDValue Data = N->getValue();
2385 EVT MemoryVT = N->getMemoryVT();
2386 unsigned Alignment = N->getOriginalAlignment();
2387 SDLoc DL(N);
2388
2389 // Split all operands
2390 EVT LoMemVT, HiMemVT;
2391 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2392
2393 SDValue DataLo, DataHi;
2394 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2395 // Split Data operand
2396 GetSplitVector(Data, DataLo, DataHi);
2397 else
2398 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2399
2400 SDValue MaskLo, MaskHi;
2401 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2402 // Split Mask operand
2403 GetSplitVector(Mask, MaskLo, MaskHi);
2404 else
2405 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2406
2407 SDValue IndexHi, IndexLo;
2408 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2409 GetSplitVector(Index, IndexLo, IndexHi);
2410 else
2411 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2412
2413 SDValue Lo;
2414 MachineMemOperand *MMO = DAG.getMachineFunction().
2415 getMachineMemOperand(N->getPointerInfo(),
2416 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2417 Alignment, N->getAAInfo(), N->getRanges());
2418
2419 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2420 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2421 DL, OpsLo, MMO);
2422
2423 MMO = DAG.getMachineFunction().
2424 getMachineMemOperand(N->getPointerInfo(),
2425 MachineMemOperand::MOStore, HiMemVT.getStoreSize(),
2426 Alignment, N->getAAInfo(), N->getRanges());
2427
2428 // The order of the Scatter operation after split is well defined. The "Hi"
2429 // part comes after the "Lo". So these two operations should be chained one
2430 // after another.
2431 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2432 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2433 DL, OpsHi, MMO);
2434 }
2435
2436 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2437 assert(N->isUnindexed() && "Indexed store of vector?");
2438 assert(OpNo == 1 && "Can only split the stored value");
2439 SDLoc DL(N);
2440
2441 bool isTruncating = N->isTruncatingStore();
2442 SDValue Ch = N->getChain();
2443 SDValue Ptr = N->getBasePtr();
2444 EVT MemoryVT = N->getMemoryVT();
2445 unsigned Alignment = N->getOriginalAlignment();
2446 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2447 AAMDNodes AAInfo = N->getAAInfo();
2448 SDValue Lo, Hi;
2449 GetSplitVector(N->getOperand(1), Lo, Hi);
2450
2451 EVT LoMemVT, HiMemVT;
2452 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2453
2454 // Scalarize if the split halves are not byte-sized.
2455 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2456 return TLI.scalarizeVectorStore(N, DAG);
2457
2458 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2459
2460 if (isTruncating)
2461 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2462 Alignment, MMOFlags, AAInfo);
2463 else
2464 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2465 AAInfo);
2466
2467 // Increment the pointer to the other half.
2468 Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2469
2470 if (isTruncating)
2471 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2472 N->getPointerInfo().getWithOffset(IncrementSize),
2473 HiMemVT, Alignment, MMOFlags, AAInfo);
2474 else
2475 Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2476 N->getPointerInfo().getWithOffset(IncrementSize),
2477 Alignment, MMOFlags, AAInfo);
2478
2479 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2480 }
2481
2482 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2483 SDLoc DL(N);
2484
2485 // The input operands all must have the same type, and we know the result
2486 // type is valid. Convert this to a buildvector which extracts all the
2487 // input elements.
2488 // TODO: If the input elements are power-two vectors, we could convert this to
2489 // a new CONCAT_VECTORS node with elements that are half-wide.
2490 SmallVector<SDValue, 32> Elts;
2491 EVT EltVT = N->getValueType(0).getVectorElementType();
2492 for (const SDValue &Op : N->op_values()) {
2493 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2494 i != e; ++i) {
2495 Elts.push_back(DAG.getNode(
2496 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2497 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2498 }
2499 }
2500
2501 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2502 }
2503
2504 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2505 // The result type is legal, but the input type is illegal. If splitting
2506 // ends up with the result type of each half still being legal, just
2507 // do that. If, however, that would result in an illegal result type,
2508 // we can try to get more clever with power-two vectors. Specifically,
2509 // split the input type, but also widen the result element size, then
2510 // concatenate the halves and truncate again. For example, consider a target
2511 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2512 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2513 // %inlo = v4i32 extract_subvector %in, 0
2514 // %inhi = v4i32 extract_subvector %in, 4
2515 // %lo16 = v4i16 trunc v4i32 %inlo
2516 // %hi16 = v4i16 trunc v4i32 %inhi
2517 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2518 // %res = v8i8 trunc v8i16 %in16
2519 //
2520 // Without this transform, the original truncate would end up being
2521 // scalarized, which is pretty much always a last resort.
2522 SDValue InVec = N->getOperand(0);
2523 EVT InVT = InVec->getValueType(0);
2524 EVT OutVT = N->getValueType(0);
2525 unsigned NumElements = OutVT.getVectorNumElements();
2526 bool IsFloat = OutVT.isFloatingPoint();
2527
2528 // Widening should have already made sure this is a power-two vector
2529 // if we're trying to split it at all. assert() that's true, just in case.
2530 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2531
2532 unsigned InElementSize = InVT.getScalarSizeInBits();
2533 unsigned OutElementSize = OutVT.getScalarSizeInBits();
2534
2535 // Determine the split output VT. If its legal we can just split dirctly.
2536 EVT LoOutVT, HiOutVT;
2537 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2538 assert(LoOutVT == HiOutVT && "Unequal split?");
2539
2540 // If the input elements are only 1/2 the width of the result elements,
2541 // just use the normal splitting. Our trick only work if there's room
2542 // to split more than once.
2543 if (isTypeLegal(LoOutVT) ||
2544 InElementSize <= OutElementSize * 2)
2545 return SplitVecOp_UnaryOp(N);
2546 SDLoc DL(N);
2547
2548 // Don't touch if this will be scalarized.
2549 EVT FinalVT = InVT;
2550 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2551 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2552
2553 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2554 return SplitVecOp_UnaryOp(N);
2555
2556 // Get the split input vector.
2557 SDValue InLoVec, InHiVec;
2558 GetSplitVector(InVec, InLoVec, InHiVec);
2559
2560 // Truncate them to 1/2 the element size.
2561 EVT HalfElementVT = IsFloat ?
2562 EVT::getFloatingPointVT(InElementSize/2) :
2563 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2564 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2565 NumElements/2);
2566 SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2567 SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2568 // Concatenate them to get the full intermediate truncation result.
2569 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2570 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2571 HalfHi);
2572 // Now finish up by truncating all the way down to the original result
2573 // type. This should normally be something that ends up being legal directly,
2574 // but in theory if a target has very wide vectors and an annoyingly
2575 // restricted set of legal types, this split can chain to build things up.
2576 return IsFloat
2577 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2578 DAG.getTargetConstant(
2579 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2580 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2581 }
2582
2583 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2584 assert(N->getValueType(0).isVector() &&
2585 N->getOperand(0).getValueType().isVector() &&
2586 "Operand types must be vectors");
2587 // The result has a legal vector type, but the input needs splitting.
2588 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2589 SDLoc DL(N);
2590 GetSplitVector(N->getOperand(0), Lo0, Hi0);
2591 GetSplitVector(N->getOperand(1), Lo1, Hi1);
2592 unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2593 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2594 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2595
2596 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2597 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2598 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2599 return PromoteTargetBoolean(Con, N->getValueType(0));
2600 }
2601
2602
2603 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2604 // The result has a legal vector type, but the input needs splitting.
2605 EVT ResVT = N->getValueType(0);
2606 SDValue Lo, Hi;
2607 SDLoc DL(N);
2608 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2609 EVT InVT = Lo.getValueType();
2610
2611 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2612 InVT.getVectorNumElements());
2613
2614 if (N->isStrictFPOpcode()) {
2615 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2616 { N->getOperand(0), Lo, N->getOperand(2) });
2617 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2618 { N->getOperand(0), Hi, N->getOperand(2) });
2619 // Legalize the chain result - switch anything that used the old chain to
2620 // use the new one.
2621 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2622 Lo.getValue(1), Hi.getValue(1));
2623 ReplaceValueWith(SDValue(N, 1), NewChain);
2624 } else {
2625 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2626 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2627 }
2628
2629 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2630 }
2631
2632 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2633 // The result (and the first input) has a legal vector type, but the second
2634 // input needs splitting.
2635 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2636 }
2637
2638
2639 //===----------------------------------------------------------------------===//
2640 // Result Vector Widening
2641 //===----------------------------------------------------------------------===//
2642
2643 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2644 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2645 dbgs() << "\n");
2646
2647 // See if the target wants to custom widen this node.
2648 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2649 return;
2650
2651 SDValue Res = SDValue();
2652 switch (N->getOpcode()) {
2653 default:
2654 #ifndef NDEBUG
2655 dbgs() << "WidenVectorResult #" << ResNo << ": ";
2656 N->dump(&DAG);
2657 dbgs() << "\n";
2658 #endif
2659 llvm_unreachable("Do not know how to widen the result of this operator!");
2660
2661 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2662 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2663 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2664 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2665 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2666 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2667 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2668 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2669 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2670 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2671 case ISD::VSELECT:
2672 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2673 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2674 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2675 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2676 case ISD::VECTOR_SHUFFLE:
2677 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2678 break;
2679 case ISD::MLOAD:
2680 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2681 break;
2682 case ISD::MGATHER:
2683 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2684 break;
2685
2686 case ISD::ADD:
2687 case ISD::AND:
2688 case ISD::MUL:
2689 case ISD::MULHS:
2690 case ISD::MULHU:
2691 case ISD::OR:
2692 case ISD::SUB:
2693 case ISD::XOR:
2694 case ISD::FMINNUM:
2695 case ISD::FMAXNUM:
2696 case ISD::FMINIMUM:
2697 case ISD::FMAXIMUM:
2698 case ISD::SMIN:
2699 case ISD::SMAX:
2700 case ISD::UMIN:
2701 case ISD::UMAX:
2702 case ISD::UADDSAT:
2703 case ISD::SADDSAT:
2704 case ISD::USUBSAT:
2705 case ISD::SSUBSAT:
2706 Res = WidenVecRes_Binary(N);
2707 break;
2708
2709 case ISD::FADD:
2710 case ISD::FMUL:
2711 case ISD::FPOW:
2712 case ISD::FSUB:
2713 case ISD::FDIV:
2714 case ISD::FREM:
2715 case ISD::SDIV:
2716 case ISD::UDIV:
2717 case ISD::SREM:
2718 case ISD::UREM:
2719 Res = WidenVecRes_BinaryCanTrap(N);
2720 break;
2721
2722 case ISD::STRICT_FADD:
2723 case ISD::STRICT_FSUB:
2724 case ISD::STRICT_FMUL:
2725 case ISD::STRICT_FDIV:
2726 case ISD::STRICT_FREM:
2727 case ISD::STRICT_FSQRT:
2728 case ISD::STRICT_FMA:
2729 case ISD::STRICT_FPOW:
2730 case ISD::STRICT_FPOWI:
2731 case ISD::STRICT_FSIN:
2732 case ISD::STRICT_FCOS:
2733 case ISD::STRICT_FEXP:
2734 case ISD::STRICT_FEXP2:
2735 case ISD::STRICT_FLOG:
2736 case ISD::STRICT_FLOG10:
2737 case ISD::STRICT_FLOG2:
2738 case ISD::STRICT_FRINT:
2739 case ISD::STRICT_FNEARBYINT:
2740 case ISD::STRICT_FMAXNUM:
2741 case ISD::STRICT_FMINNUM:
2742 case ISD::STRICT_FCEIL:
2743 case ISD::STRICT_FFLOOR:
2744 case ISD::STRICT_FROUND:
2745 case ISD::STRICT_FTRUNC:
2746 Res = WidenVecRes_StrictFP(N);
2747 break;
2748
2749 case ISD::UADDO:
2750 case ISD::SADDO:
2751 case ISD::USUBO:
2752 case ISD::SSUBO:
2753 case ISD::UMULO:
2754 case ISD::SMULO:
2755 Res = WidenVecRes_OverflowOp(N, ResNo);
2756 break;
2757
2758 case ISD::FCOPYSIGN:
2759 Res = WidenVecRes_FCOPYSIGN(N);
2760 break;
2761
2762 case ISD::FPOWI:
2763 Res = WidenVecRes_POWI(N);
2764 break;
2765
2766 case ISD::SHL:
2767 case ISD::SRA:
2768 case ISD::SRL:
2769 Res = WidenVecRes_Shift(N);
2770 break;
2771
2772 case ISD::ANY_EXTEND_VECTOR_INREG:
2773 case ISD::SIGN_EXTEND_VECTOR_INREG:
2774 case ISD::ZERO_EXTEND_VECTOR_INREG:
2775 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2776 break;
2777
2778 case ISD::ANY_EXTEND:
2779 case ISD::FP_EXTEND:
2780 case ISD::FP_ROUND:
2781 case ISD::FP_TO_SINT:
2782 case ISD::FP_TO_UINT:
2783 case ISD::SIGN_EXTEND:
2784 case ISD::SINT_TO_FP:
2785 case ISD::TRUNCATE:
2786 case ISD::UINT_TO_FP:
2787 case ISD::ZERO_EXTEND:
2788 Res = WidenVecRes_Convert(N);
2789 break;
2790
2791 case ISD::STRICT_FP_EXTEND:
2792 case ISD::STRICT_FP_ROUND:
2793 Res = WidenVecRes_Convert_StrictFP(N);
2794 break;
2795
2796 case ISD::FABS:
2797 case ISD::FCEIL:
2798 case ISD::FCOS:
2799 case ISD::FEXP:
2800 case ISD::FEXP2:
2801 case ISD::FFLOOR:
2802 case ISD::FLOG:
2803 case ISD::FLOG10:
2804 case ISD::FLOG2:
2805 case ISD::FNEARBYINT:
2806 case ISD::FRINT:
2807 case ISD::FROUND:
2808 case ISD::FSIN:
2809 case ISD::FSQRT:
2810 case ISD::FTRUNC: {
2811 // We're going to widen this vector op to a legal type by padding with undef
2812 // elements. If the wide vector op is eventually going to be expanded to
2813 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2814 // libcalls on the undef elements.
2815 EVT VT = N->getValueType(0);
2816 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2817 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2818 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2819 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2820 break;
2821 }
2822 }
2823 // If the target has custom/legal support for the scalar FP intrinsic ops
2824 // (they are probably not destined to become libcalls), then widen those like
2825 // any other unary ops.
2826 LLVM_FALLTHROUGH;
2827
2828 case ISD::ABS:
2829 case ISD::BITREVERSE:
2830 case ISD::BSWAP:
2831 case ISD::CTLZ:
2832 case ISD::CTLZ_ZERO_UNDEF:
2833 case ISD::CTPOP:
2834 case ISD::CTTZ:
2835 case ISD::CTTZ_ZERO_UNDEF:
2836 case ISD::FNEG:
2837 case ISD::FCANONICALIZE:
2838 Res = WidenVecRes_Unary(N);
2839 break;
2840 case ISD::FMA:
2841 Res = WidenVecRes_Ternary(N);
2842 break;
2843 }
2844
2845 // If Res is null, the sub-method took care of registering the result.
2846 if (Res.getNode())
2847 SetWidenedVector(SDValue(N, ResNo), Res);
2848 }
2849
2850 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2851 // Ternary op widening.
2852 SDLoc dl(N);
2853 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2854 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2855 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2856 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2857 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2858 }
2859
2860 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2861 // Binary op widening.
2862 SDLoc dl(N);
2863 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2864 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2865 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2866 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2867 }
2868
2869 // Given a vector of operations that have been broken up to widen, see
2870 // if we can collect them together into the next widest legal VT. This
2871 // implementation is trap-safe.
2872 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2873 SmallVectorImpl<SDValue> &ConcatOps,
2874 unsigned ConcatEnd, EVT VT, EVT MaxVT,
2875 EVT WidenVT) {
2876 // Check to see if we have a single operation with the widen type.
2877 if (ConcatEnd == 1) {
2878 VT = ConcatOps[0].getValueType();
2879 if (VT == WidenVT)
2880 return ConcatOps[0];
2881 }
2882
2883 SDLoc dl(ConcatOps[0]);
2884 EVT WidenEltVT = WidenVT.getVectorElementType();
2885
2886 // while (Some element of ConcatOps is not of type MaxVT) {
2887 // From the end of ConcatOps, collect elements of the same type and put
2888 // them into an op of the next larger supported type
2889 // }
2890 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2891 int Idx = ConcatEnd - 1;
2892 VT = ConcatOps[Idx--].getValueType();
2893 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2894 Idx--;
2895
2896 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2897 EVT NextVT;
2898 do {
2899 NextSize *= 2;
2900 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2901 } while (!TLI.isTypeLegal(NextVT));
2902
2903 if (!VT.isVector()) {
2904 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2905 SDValue VecOp = DAG.getUNDEF(NextVT);
2906 unsigned NumToInsert = ConcatEnd - Idx - 1;
2907 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2908 VecOp = DAG.getNode(
2909 ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2910 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2911 }
2912 ConcatOps[Idx+1] = VecOp;
2913 ConcatEnd = Idx + 2;
2914 } else {
2915 // Vector type, create a CONCAT_VECTORS of type NextVT
2916 SDValue undefVec = DAG.getUNDEF(VT);
2917 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2918 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2919 unsigned RealVals = ConcatEnd - Idx - 1;
2920 unsigned SubConcatEnd = 0;
2921 unsigned SubConcatIdx = Idx + 1;
2922 while (SubConcatEnd < RealVals)
2923 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2924 while (SubConcatEnd < OpsToConcat)
2925 SubConcatOps[SubConcatEnd++] = undefVec;
2926 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2927 NextVT, SubConcatOps);
2928 ConcatEnd = SubConcatIdx + 1;
2929 }
2930 }
2931
2932 // Check to see if we have a single operation with the widen type.
2933 if (ConcatEnd == 1) {
2934 VT = ConcatOps[0].getValueType();
2935 if (VT == WidenVT)
2936 return ConcatOps[0];
2937 }
2938
2939 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2940 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2941 if (NumOps != ConcatEnd ) {
2942 SDValue UndefVal = DAG.getUNDEF(MaxVT);
2943 for (unsigned j = ConcatEnd; j < NumOps; ++j)
2944 ConcatOps[j] = UndefVal;
2945 }
2946 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2947 makeArrayRef(ConcatOps.data(), NumOps));
2948 }
2949
2950 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2951 // Binary op widening for operations that can trap.
2952 unsigned Opcode = N->getOpcode();
2953 SDLoc dl(N);
2954 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2955 EVT WidenEltVT = WidenVT.getVectorElementType();
2956 EVT VT = WidenVT;
2957 unsigned NumElts = VT.getVectorNumElements();
2958 const SDNodeFlags Flags = N->getFlags();
2959 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2960 NumElts = NumElts / 2;
2961 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2962 }
2963
2964 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2965 // Operation doesn't trap so just widen as normal.
2966 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2967 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2968 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2969 }
2970
2971 // No legal vector version so unroll the vector operation and then widen.
2972 if (NumElts == 1)
2973 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2974
2975 // Since the operation can trap, apply operation on the original vector.
2976 EVT MaxVT = VT;
2977 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2978 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2979 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2980
2981 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2982 unsigned ConcatEnd = 0; // Current ConcatOps index.
2983 int Idx = 0; // Current Idx into input vectors.
2984
2985 // NumElts := greatest legal vector size (at most WidenVT)
2986 // while (orig. vector has unhandled elements) {
2987 // take munches of size NumElts from the beginning and add to ConcatOps
2988 // NumElts := next smaller supported vector size or 1
2989 // }
2990 while (CurNumElts != 0) {
2991 while (CurNumElts >= NumElts) {
2992 SDValue EOp1 = DAG.getNode(
2993 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2994 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2995 SDValue EOp2 = DAG.getNode(
2996 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2997 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2998 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2999 Idx += NumElts;
3000 CurNumElts -= NumElts;
3001 }
3002 do {
3003 NumElts = NumElts / 2;
3004 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3005 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3006
3007 if (NumElts == 1) {
3008 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3009 SDValue EOp1 = DAG.getNode(
3010 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
3011 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3012 SDValue EOp2 = DAG.getNode(
3013 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
3014 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3015 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3016 EOp1, EOp2, Flags);
3017 }
3018 CurNumElts = 0;
3019 }
3020 }
3021
3022 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3023 }
3024
3025 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3026 // StrictFP op widening for operations that can trap.
3027 unsigned NumOpers = N->getNumOperands();
3028 unsigned Opcode = N->getOpcode();
3029 SDLoc dl(N);
3030 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3031 EVT WidenEltVT = WidenVT.getVectorElementType();
3032 EVT VT = WidenVT;
3033 unsigned NumElts = VT.getVectorNumElements();
3034 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3035 NumElts = NumElts / 2;
3036 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3037 }
3038
3039 // No legal vector version so unroll the vector operation and then widen.
3040 if (NumElts == 1)
3041 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3042
3043 // Since the operation can trap, apply operation on the original vector.
3044 EVT MaxVT = VT;
3045 SmallVector<SDValue, 4> InOps;
3046 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3047
3048 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3049 SmallVector<SDValue, 16> Chains;
3050 unsigned ConcatEnd = 0; // Current ConcatOps index.
3051 int Idx = 0; // Current Idx into input vectors.
3052
3053 // The Chain is the first operand.
3054 InOps.push_back(N->getOperand(0));
3055
3056 // Now process the remaining operands.
3057 for (unsigned i = 1; i < NumOpers; ++i) {
3058 SDValue Oper = N->getOperand(i);
3059
3060 if (Oper.getValueType().isVector()) {
3061 assert(Oper.getValueType() == N->getValueType(0) &&
3062 "Invalid operand type to widen!");
3063 Oper = GetWidenedVector(Oper);
3064 }
3065
3066 InOps.push_back(Oper);
3067 }
3068
3069 // NumElts := greatest legal vector size (at most WidenVT)
3070 // while (orig. vector has unhandled elements) {
3071 // take munches of size NumElts from the beginning and add to ConcatOps
3072 // NumElts := next smaller supported vector size or 1
3073 // }
3074 while (CurNumElts != 0) {
3075 while (CurNumElts >= NumElts) {
3076 SmallVector<SDValue, 4> EOps;
3077
3078 for (unsigned i = 0; i < NumOpers; ++i) {
3079 SDValue Op = InOps[i];
3080
3081 if (Op.getValueType().isVector())
3082 Op = DAG.getNode(
3083 ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3084 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3085
3086 EOps.push_back(Op);
3087 }
3088
3089 EVT OperVT[] = {VT, MVT::Other};
3090 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3091 ConcatOps[ConcatEnd++] = Oper;
3092 Chains.push_back(Oper.getValue(1));
3093 Idx += NumElts;
3094 CurNumElts -= NumElts;
3095 }
3096 do {
3097 NumElts = NumElts / 2;
3098 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3099 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3100
3101 if (NumElts == 1) {
3102 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3103 SmallVector<SDValue, 4> EOps;
3104
3105 for (unsigned i = 0; i < NumOpers; ++i) {
3106 SDValue Op = InOps[i];
3107
3108 if (Op.getValueType().isVector())
3109 Op = DAG.getNode(
3110 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3111 DAG.getConstant(Idx, dl,
3112 TLI.getVectorIdxTy(DAG.getDataLayout())));
3113
3114 EOps.push_back(Op);
3115 }
3116
3117 EVT WidenVT[] = {WidenEltVT, MVT::Other};
3118 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3119 ConcatOps[ConcatEnd++] = Oper;
3120 Chains.push_back(Oper.getValue(1));
3121 }
3122 CurNumElts = 0;
3123 }
3124 }
3125
3126 // Build a factor node to remember all the Ops that have been created.
3127 SDValue NewChain;
3128 if (Chains.size() == 1)
3129 NewChain = Chains[0];
3130 else
3131 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3132 ReplaceValueWith(SDValue(N, 1), NewChain);
3133
3134 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3135 }
3136
3137 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3138 SDLoc DL(N);
3139 EVT ResVT = N->getValueType(0);
3140 EVT OvVT = N->getValueType(1);
3141 EVT WideResVT, WideOvVT;
3142 SDValue WideLHS, WideRHS;
3143
3144 // TODO: This might result in a widen/split loop.
3145 if (ResNo == 0) {
3146 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3147 WideOvVT = EVT::getVectorVT(
3148 *DAG.getContext(), OvVT.getVectorElementType(),
3149 WideResVT.getVectorNumElements());
3150
3151 WideLHS = GetWidenedVector(N->getOperand(0));
3152 WideRHS = GetWidenedVector(N->getOperand(1));
3153 } else {
3154 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3155 WideResVT = EVT::getVectorVT(
3156 *DAG.getContext(), ResVT.getVectorElementType(),
3157 WideOvVT.getVectorNumElements());
3158
3159 SDValue Zero = DAG.getConstant(
3160 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3161 WideLHS = DAG.getNode(
3162 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3163 N->getOperand(0), Zero);
3164 WideRHS = DAG.getNode(
3165 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3166 N->getOperand(1), Zero);
3167 }
3168
3169 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3170 SDNode *WideNode = DAG.getNode(
3171 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3172
3173 // Replace the other vector result not being explicitly widened here.
3174 unsigned OtherNo = 1 - ResNo;
3175 EVT OtherVT = N->getValueType(OtherNo);
3176 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3177 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3178 } else {
3179 SDValue Zero = DAG.getConstant(
3180 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3181 SDValue OtherVal = DAG.getNode(
3182 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3183 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3184 }
3185
3186 return SDValue(WideNode, ResNo);
3187 }
3188
3189 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3190 SDValue InOp = N->getOperand(0);
3191 SDLoc DL(N);
3192
3193 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3194 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3195
3196 EVT InVT = InOp.getValueType();
3197 EVT InEltVT = InVT.getVectorElementType();
3198 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3199
3200 unsigned Opcode = N->getOpcode();
3201 unsigned InVTNumElts = InVT.getVectorNumElements();
3202 const SDNodeFlags Flags = N->getFlags();
3203 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3204 InOp = GetWidenedVector(N->getOperand(0));
3205 InVT = InOp.getValueType();
3206 InVTNumElts = InVT.getVectorNumElements();
3207 if (InVTNumElts == WidenNumElts) {
3208 if (N->getNumOperands() == 1)
3209 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3210 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3211 }
3212 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3213 // If both input and result vector types are of same width, extend
3214 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3215 // accepts fewer elements in the result than in the input.
3216 if (Opcode == ISD::ANY_EXTEND)
3217 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3218 if (Opcode == ISD::SIGN_EXTEND)
3219 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3220 if (Opcode == ISD::ZERO_EXTEND)
3221 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3222 }
3223 }
3224
3225 if (TLI.isTypeLegal(InWidenVT)) {
3226 // Because the result and the input are different vector types, widening
3227 // the result could create a legal type but widening the input might make
3228 // it an illegal type that might lead to repeatedly splitting the input
3229 // and then widening it. To avoid this, we widen the input only if
3230 // it results in a legal type.
3231 if (WidenNumElts % InVTNumElts == 0) {
3232 // Widen the input and call convert on the widened input vector.
3233 unsigned NumConcat = WidenNumElts/InVTNumElts;
3234 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3235 Ops[0] = InOp;
3236 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3237 if (N->getNumOperands() == 1)
3238 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3239 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3240 }
3241
3242 if (InVTNumElts % WidenNumElts == 0) {
3243 SDValue InVal = DAG.getNode(
3244 ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3245 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3246 // Extract the input and convert the shorten input vector.
3247 if (N->getNumOperands() == 1)
3248 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3249 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3250 }
3251 }
3252
3253 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3254 EVT EltVT = WidenVT.getVectorElementType();
3255 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3256 // Use the original element count so we don't do more scalar opts than
3257 // necessary.
3258 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3259 for (unsigned i=0; i < MinElts; ++i) {
3260 SDValue Val = DAG.getNode(
3261 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3262 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3263 if (N->getNumOperands() == 1)
3264 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3265 else
3266 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3267 }
3268
3269 return DAG.getBuildVector(WidenVT, DL, Ops);
3270 }
3271
3272 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3273 SDValue InOp = N->getOperand(1);
3274 SDLoc DL(N);
3275 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3276
3277 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3278 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3279 SmallVector<EVT, 2> WidenVTs = { WidenVT, MVT::Other };
3280
3281 EVT InVT = InOp.getValueType();
3282 EVT InEltVT = InVT.getVectorElementType();
3283
3284 unsigned Opcode = N->getOpcode();
3285
3286 // FIXME: Optimizations need to be implemented here.
3287
3288 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3289 EVT EltVT = WidenVT.getVectorElementType();
3290 SmallVector<EVT, 2> EltVTs = { EltVT, MVT::Other };
3291 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3292 SmallVector<SDValue, 32> OpChains;
3293 // Use the original element count so we don't do more scalar opts than
3294 // necessary.
3295 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3296 for (unsigned i=0; i < MinElts; ++i) {
3297 NewOps[1] = DAG.getNode(
3298 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3299 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3300 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3301 OpChains.push_back(Ops[i].getValue(1));
3302 }
3303 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3304 ReplaceValueWith(SDValue(N, 1), NewChain);
3305
3306 return DAG.getBuildVector(WidenVT, DL, Ops);
3307 }
3308
3309 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3310 unsigned Opcode = N->getOpcode();
3311 SDValue InOp = N->getOperand(0);
3312 SDLoc DL(N);
3313
3314 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3315 EVT WidenSVT = WidenVT.getVectorElementType();
3316 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3317
3318 EVT InVT = InOp.getValueType();
3319 EVT InSVT = InVT.getVectorElementType();
3320 unsigned InVTNumElts = InVT.getVectorNumElements();
3321
3322 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3323 InOp = GetWidenedVector(InOp);
3324 InVT = InOp.getValueType();
3325 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3326 switch (Opcode) {
3327 case ISD::ANY_EXTEND_VECTOR_INREG:
3328 case ISD::SIGN_EXTEND_VECTOR_INREG:
3329 case ISD::ZERO_EXTEND_VECTOR_INREG:
3330 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3331 }
3332 }
3333 }
3334
3335 // Unroll, extend the scalars and rebuild the vector.
3336 SmallVector<SDValue, 16> Ops;
3337 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3338 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3339 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3340 switch (Opcode) {
3341 case ISD::ANY_EXTEND_VECTOR_INREG:
3342 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3343 break;
3344 case ISD::SIGN_EXTEND_VECTOR_INREG:
3345 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3346 break;
3347 case ISD::ZERO_EXTEND_VECTOR_INREG:
3348 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3349 break;
3350 default:
3351 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3352 }
3353 Ops.push_back(Val);
3354 }
3355
3356 while (Ops.size() != WidenNumElts)
3357 Ops.push_back(DAG.getUNDEF(WidenSVT));
3358
3359 return DAG.getBuildVector(WidenVT, DL, Ops);
3360 }
3361
3362 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3363 // If this is an FCOPYSIGN with same input types, we can treat it as a
3364 // normal (can trap) binary op.
3365 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3366 return WidenVecRes_BinaryCanTrap(N);
3367
3368 // If the types are different, fall back to unrolling.
3369 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3370 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3371 }
3372
3373 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3374 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3375 SDValue InOp = GetWidenedVector(N->getOperand(0));
3376 SDValue ShOp = N->getOperand(1);
3377 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3378 }
3379
3380 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3381 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3382 SDValue InOp = GetWidenedVector(N->getOperand(0));
3383 SDValue ShOp = N->getOperand(1);
3384
3385 EVT ShVT = ShOp.getValueType();
3386 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
3387 ShOp = GetWidenedVector(ShOp);
3388 ShVT = ShOp.getValueType();
3389 }
3390 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3391 ShVT.getVectorElementType(),
3392 WidenVT.getVectorNumElements());
3393 if (ShVT != ShWidenVT)
3394 ShOp = ModifyToType(ShOp, ShWidenVT);
3395
3396 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3397 }
3398
3399 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3400 // Unary op widening.
3401 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3402 SDValue InOp = GetWidenedVector(N->getOperand(0));
3403 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3404 }
3405
3406 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3407 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3408 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3409 cast<VTSDNode>(N->getOperand(1))->getVT()
3410 .getVectorElementType(),
3411 WidenVT.getVectorNumElements());
3412 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3413 return DAG.getNode(N->getOpcode(), SDLoc(N),
3414 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3415 }
3416
3417 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3418 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3419 return GetWidenedVector(WidenVec);
3420 }
3421
3422 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3423 SDValue InOp = N->getOperand(0);
3424 EVT InVT = InOp.getValueType();
3425 EVT VT = N->getValueType(0);
3426 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3427 SDLoc dl(N);
3428
3429 switch (getTypeAction(InVT)) {
3430 case TargetLowering::TypeLegal:
3431 break;
3432 case TargetLowering::TypePromoteInteger:
3433 // If the incoming type is a vector that is being promoted, then
3434 // we know that the elements are arranged differently and that we
3435 // must perform the conversion using a stack slot.
3436 if (InVT.isVector())
3437 break;
3438
3439 // If the InOp is promoted to the same size, convert it. Otherwise,
3440 // fall out of the switch and widen the promoted input.
3441 InOp = GetPromotedInteger(InOp);
3442 InVT = InOp.getValueType();
3443 if (WidenVT.bitsEq(InVT))
3444 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3445 break;
3446 case TargetLowering::TypeSoftenFloat:
3447 case TargetLowering::TypePromoteFloat:
3448 case TargetLowering::TypeExpandInteger:
3449 case TargetLowering::TypeExpandFloat:
3450 case TargetLowering::TypeScalarizeVector:
3451 case TargetLowering::TypeSplitVector:
3452 break;
3453 case TargetLowering::TypeWidenVector:
3454 // If the InOp is widened to the same size, convert it. Otherwise, fall
3455 // out of the switch and widen the widened input.
3456 InOp = GetWidenedVector(InOp);
3457 InVT = InOp.getValueType();
3458 if (WidenVT.bitsEq(InVT))
3459 // The input widens to the same size. Convert to the widen value.
3460 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3461 break;
3462 }
3463
3464 unsigned WidenSize = WidenVT.getSizeInBits();
3465 unsigned InSize = InVT.getSizeInBits();
3466 // x86mmx is not an acceptable vector element type, so don't try.
3467 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3468 // Determine new input vector type. The new input vector type will use
3469 // the same element type (if its a vector) or use the input type as a
3470 // vector. It is the same size as the type to widen to.
3471 EVT NewInVT;
3472 unsigned NewNumElts = WidenSize / InSize;
3473 if (InVT.isVector()) {
3474 EVT InEltVT = InVT.getVectorElementType();
3475 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3476 WidenSize / InEltVT.getSizeInBits());
3477 } else {
3478 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3479 }
3480
3481 if (TLI.isTypeLegal(NewInVT)) {
3482 SDValue NewVec;
3483 if (InVT.isVector()) {
3484 // Because the result and the input are different vector types, widening
3485 // the result could create a legal type but widening the input might make
3486 // it an illegal type that might lead to repeatedly splitting the input
3487 // and then widening it. To avoid this, we widen the input only if
3488 // it results in a legal type.
3489 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3490 Ops[0] = InOp;
3491
3492 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3493 } else {
3494 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3495 }
3496 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3497 }
3498 }
3499
3500 return CreateStackStoreLoad(InOp, WidenVT);
3501 }
3502
3503 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3504 SDLoc dl(N);
3505 // Build a vector with undefined for the new nodes.
3506 EVT VT = N->getValueType(0);
3507
3508 // Integer BUILD_VECTOR operands may be larger than the node's vector element
3509 // type. The UNDEFs need to have the same type as the existing operands.
3510 EVT EltVT = N->getOperand(0).getValueType();
3511 unsigned NumElts = VT.getVectorNumElements();
3512
3513 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3514 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3515
3516 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3517 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3518 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3519
3520 return DAG.getBuildVector(WidenVT, dl, NewOps);
3521 }
3522
3523 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3524 EVT InVT = N->getOperand(0).getValueType();
3525 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3526 SDLoc dl(N);
3527 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3528 unsigned NumInElts = InVT.getVectorNumElements();
3529 unsigned NumOperands = N->getNumOperands();
3530
3531 bool InputWidened = false; // Indicates we need to widen the input.
3532 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3533 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3534 // Add undef vectors to widen to correct length.
3535 unsigned NumConcat = WidenVT.getVectorNumElements() /
3536 InVT.getVectorNumElements();
3537 SDValue UndefVal = DAG.getUNDEF(InVT);
3538 SmallVector<SDValue, 16> Ops(NumConcat);
3539 for (unsigned i=0; i < NumOperands; ++i)
3540 Ops[i] = N->getOperand(i);
3541 for (unsigned i = NumOperands; i != NumConcat; ++i)
3542 Ops[i] = UndefVal;
3543 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3544 }
3545 } else {
3546 InputWidened = true;
3547 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3548 // The inputs and the result are widen to the same value.
3549 unsigned i;
3550 for (i=1; i < NumOperands; ++i)
3551 if (!N->getOperand(i).isUndef())
3552 break;
3553
3554 if (i == NumOperands)
3555 // Everything but the first operand is an UNDEF so just return the
3556 // widened first operand.
3557 return GetWidenedVector(N->getOperand(0));
3558
3559 if (NumOperands == 2) {
3560 // Replace concat of two operands with a shuffle.
3561 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3562 for (unsigned i = 0; i < NumInElts; ++i) {
3563 MaskOps[i] = i;
3564 MaskOps[i + NumInElts] = i + WidenNumElts;
3565 }
3566 return DAG.getVectorShuffle(WidenVT, dl,
3567 GetWidenedVector(N->getOperand(0)),
3568 GetWidenedVector(N->getOperand(1)),
3569 MaskOps);
3570 }
3571 }
3572 }
3573
3574 // Fall back to use extracts and build vector.
3575 EVT EltVT = WidenVT.getVectorElementType();
3576 SmallVector<SDValue, 16> Ops(WidenNumElts);
3577 unsigned Idx = 0;
3578 for (unsigned i=0; i < NumOperands; ++i) {
3579 SDValue InOp = N->getOperand(i);
3580 if (InputWidened)
3581 InOp = GetWidenedVector(InOp);
3582 for (unsigned j=0; j < NumInElts; ++j)
3583 Ops[Idx++] = DAG.getNode(
3584 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3585 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3586 }
3587 SDValue UndefVal = DAG.getUNDEF(EltVT);
3588 for (; Idx < WidenNumElts; ++Idx)
3589 Ops[Idx] = UndefVal;
3590 return DAG.getBuildVector(WidenVT, dl, Ops);
3591 }
3592
3593 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3594 EVT VT = N->getValueType(0);
3595 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3596 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3597 SDValue InOp = N->getOperand(0);
3598 SDValue Idx = N->getOperand(1);
3599 SDLoc dl(N);
3600
3601 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3602 InOp = GetWidenedVector(InOp);
3603
3604 EVT InVT = InOp.getValueType();
3605
3606 // Check if we can just return the input vector after widening.
3607 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3608 if (IdxVal == 0 && InVT == WidenVT)
3609 return InOp;
3610
3611 // Check if we can extract from the vector.
3612 unsigned InNumElts = InVT.getVectorNumElements();
3613 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3614 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3615
3616 // We could try widening the input to the right length but for now, extract
3617 // the original elements, fill the rest with undefs and build a vector.
3618 SmallVector<SDValue, 16> Ops(WidenNumElts);
3619 EVT EltVT = VT.getVectorElementType();
3620 unsigned NumElts = VT.getVectorNumElements();
3621 unsigned i;
3622 for (i=0; i < NumElts; ++i)
3623 Ops[i] =
3624 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3625 DAG.getConstant(IdxVal + i, dl,
3626 TLI.getVectorIdxTy(DAG.getDataLayout())));
3627
3628 SDValue UndefVal = DAG.getUNDEF(EltVT);
3629 for (; i < WidenNumElts; ++i)
3630 Ops[i] = UndefVal;
3631 return DAG.getBuildVector(WidenVT, dl, Ops);
3632 }
3633
3634 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3635 SDValue InOp = GetWidenedVector(N->getOperand(0));
3636 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3637 InOp.getValueType(), InOp,
3638 N->getOperand(1), N->getOperand(2));
3639 }
3640
3641 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3642 LoadSDNode *LD = cast<LoadSDNode>(N);
3643 ISD::LoadExtType ExtType = LD->getExtensionType();
3644
3645 SDValue Result;
3646 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3647 if (ExtType != ISD::NON_EXTLOAD)
3648 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3649 else
3650 Result = GenWidenVectorLoads(LdChain, LD);
3651
3652 // If we generate a single load, we can use that for the chain. Otherwise,
3653 // build a factor node to remember the multiple loads are independent and
3654 // chain to that.
3655 SDValue NewChain;
3656 if (LdChain.size() == 1)
3657 NewChain = LdChain[0];
3658 else
3659 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3660
3661 // Modified the chain - switch anything that used the old chain to use
3662 // the new one.
3663 ReplaceValueWith(SDValue(N, 1), NewChain);
3664
3665 return Result;
3666 }
3667
3668 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3669
3670 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3671 SDValue Mask = N->getMask();
3672 EVT MaskVT = Mask.getValueType();
3673 SDValue PassThru = GetWidenedVector(N->getPassThru());
3674 ISD::LoadExtType ExtType = N->getExtensionType();
3675 SDLoc dl(N);
3676
3677 // The mask should be widened as well
3678 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3679 MaskVT.getVectorElementType(),
3680 WidenVT.getVectorNumElements());
3681 Mask = ModifyToType(Mask, WideMaskVT, true);
3682
3683 SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3684 Mask, PassThru, N->getMemoryVT(),
3685 N->getMemOperand(), ExtType,
3686 N->isExpandingLoad());
3687 // Legalize the chain result - switch anything that used the old chain to
3688 // use the new one.
3689 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3690 return Res;
3691 }
3692
3693 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3694
3695 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3696 SDValue Mask = N->getMask();
3697 EVT MaskVT = Mask.getValueType();
3698 SDValue PassThru = GetWidenedVector(N->getPassThru());
3699 SDValue Scale = N->getScale();
3700 unsigned NumElts = WideVT.getVectorNumElements();
3701 SDLoc dl(N);
3702
3703 // The mask should be widened as well
3704 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3705 MaskVT.getVectorElementType(),
3706 WideVT.getVectorNumElements());
3707 Mask = ModifyToType(Mask, WideMaskVT, true);
3708
3709 // Widen the Index operand
3710 SDValue Index = N->getIndex();
3711 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3712 Index.getValueType().getScalarType(),
3713 NumElts);
3714 Index = ModifyToType(Index, WideIndexVT);
3715 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3716 Scale };
3717 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3718 N->getMemoryVT(), dl, Ops,
3719 N->getMemOperand());
3720
3721 // Legalize the chain result - switch anything that used the old chain to
3722 // use the new one.
3723 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3724 return Res;
3725 }
3726
3727 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3728 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3729 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3730 WidenVT, N->getOperand(0));
3731 }
3732
3733 // Return true if this is a node that could have two SETCCs as operands.
3734 static inline bool isLogicalMaskOp(unsigned Opcode) {
3735 switch (Opcode) {
3736 case ISD::AND:
3737 case ISD::OR:
3738 case ISD::XOR:
3739 return true;
3740 }
3741 return false;
3742 }
3743
3744 // This is used just for the assert in convertMask(). Check that this either
3745 // a SETCC or a previously handled SETCC by convertMask().
3746 #ifndef NDEBUG
3747 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3748 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3749 N = N.getOperand(0);
3750 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3751 for (unsigned i = 1; i < N->getNumOperands(); ++i)
3752 if (!N->getOperand(i)->isUndef())
3753 return false;
3754 N = N.getOperand(0);
3755 }
3756
3757 if (N.getOpcode() == ISD::TRUNCATE)
3758 N = N.getOperand(0);
3759 else if (N.getOpcode() == ISD::SIGN_EXTEND)
3760 N = N.getOperand(0);
3761
3762 if (isLogicalMaskOp(N.getOpcode()))
3763 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3764 isSETCCorConvertedSETCC(N.getOperand(1));
3765
3766 return (N.getOpcode() == ISD::SETCC ||
3767 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
3768 }
3769 #endif
3770
3771 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3772 // to ToMaskVT if needed with vector extension or truncation.
3773 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3774 EVT ToMaskVT) {
3775 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3776 // FIXME: This code seems to be too restrictive, we might consider
3777 // generalizing it or dropping it.
3778 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3779
3780 // Make a new Mask node, with a legal result VT.
3781 SmallVector<SDValue, 4> Ops;
3782 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3783 Ops.push_back(InMask->getOperand(i));
3784 SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3785
3786 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3787 // extend or truncate is needed.
3788 LLVMContext &Ctx = *DAG.getContext();
3789 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3790 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3791 if (MaskScalarBits < ToMaskScalBits) {
3792 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3793 MaskVT.getVectorNumElements());
3794 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3795 } else if (MaskScalarBits > ToMaskScalBits) {
3796 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3797 MaskVT.getVectorNumElements());
3798 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3799 }
3800
3801 assert(Mask->getValueType(0).getScalarSizeInBits() ==
3802 ToMaskVT.getScalarSizeInBits() &&
3803 "Mask should have the right element size by now.");
3804
3805 // Adjust Mask to the right number of elements.
3806 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3807 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3808 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3809 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3810 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3811 ZeroIdx);
3812 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3813 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3814 EVT SubVT = Mask->getValueType(0);
3815 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3816 SubOps[0] = Mask;
3817 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3818 }
3819
3820 assert((Mask->getValueType(0) == ToMaskVT) &&
3821 "A mask of ToMaskVT should have been produced by now.");
3822
3823 return Mask;
3824 }
3825
3826 // This method tries to handle VSELECT and its mask by legalizing operands
3827 // (which may require widening) and if needed adjusting the mask vector type
3828 // to match that of the VSELECT. Without it, many cases end up with
3829 // scalarization of the SETCC, with many unnecessary instructions.
3830 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3831 LLVMContext &Ctx = *DAG.getContext();
3832 SDValue Cond = N->getOperand(0);
3833
3834 if (N->getOpcode() != ISD::VSELECT)
3835 return SDValue();
3836
3837 if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3838 return SDValue();
3839
3840 // If this is a splitted VSELECT that was previously already handled, do
3841 // nothing.
3842 EVT CondVT = Cond->getValueType(0);
3843 if (CondVT.getScalarSizeInBits() != 1)
3844 return SDValue();
3845
3846 EVT VSelVT = N->getValueType(0);
3847 // Only handle vector types which are a power of 2.
3848 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3849 return SDValue();
3850
3851 // Don't touch if this will be scalarized.
3852 EVT FinalVT = VSelVT;
3853 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3854 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3855
3856 if (FinalVT.getVectorNumElements() == 1)
3857 return SDValue();
3858
3859 // If there is support for an i1 vector mask, don't touch.
3860 if (Cond.getOpcode() == ISD::SETCC) {
3861 EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3862 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3863 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3864 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3865 if (SetCCResVT.getScalarSizeInBits() == 1)
3866 return SDValue();
3867 } else if (CondVT.getScalarType() == MVT::i1) {
3868 // If there is support for an i1 vector mask (or only scalar i1 conditions),
3869 // don't touch.
3870 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3871 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3872
3873 if (CondVT.getScalarType() == MVT::i1)
3874 return SDValue();
3875 }
3876
3877 // Get the VT and operands for VSELECT, and widen if needed.
3878 SDValue VSelOp1 = N->getOperand(1);
3879 SDValue VSelOp2 = N->getOperand(2);
3880 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3881 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3882 VSelOp1 = GetWidenedVector(VSelOp1);
3883 VSelOp2 = GetWidenedVector(VSelOp2);
3884 }
3885
3886 // The mask of the VSELECT should have integer elements.
3887 EVT ToMaskVT = VSelVT;
3888 if (!ToMaskVT.getScalarType().isInteger())
3889 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3890
3891 SDValue Mask;
3892 if (Cond->getOpcode() == ISD::SETCC) {
3893 EVT MaskVT = getSetCCResultType(Cond.getOperand(0).getValueType());
3894 Mask = convertMask(Cond, MaskVT, ToMaskVT);
3895 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3896 Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3897 Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3898 // Cond is (AND/OR/XOR (SETCC, SETCC))
3899 SDValue SETCC0 = Cond->getOperand(0);
3900 SDValue SETCC1 = Cond->getOperand(1);
3901 EVT VT0 = getSetCCResultType(SETCC0.getOperand(0).getValueType());
3902 EVT VT1 = getSetCCResultType(SETCC1.getOperand(0).getValueType());
3903 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3904 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3905 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3906 EVT MaskVT;
3907 // If the two SETCCs have different VTs, either extend/truncate one of
3908 // them to the other "towards" ToMaskVT, or truncate one and extend the
3909 // other to ToMaskVT.
3910 if (ScalarBits0 != ScalarBits1) {
3911 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3912 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3913 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3914 MaskVT = WideVT;
3915 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3916 MaskVT = NarrowVT;
3917 else
3918 MaskVT = ToMaskVT;
3919 } else
3920 // If the two SETCCs have the same VT, don't change it.
3921 MaskVT = VT0;
3922
3923 // Make new SETCCs and logical nodes.
3924 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3925 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3926 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3927
3928 // Convert the logical op for VSELECT if needed.
3929 Mask = convertMask(Cond, MaskVT, ToMaskVT);
3930 } else
3931 return SDValue();
3932
3933 return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3934 }
3935
3936 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3937 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3938 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3939
3940 SDValue Cond1 = N->getOperand(0);
3941 EVT CondVT = Cond1.getValueType();
3942 if (CondVT.isVector()) {
3943 if (SDValue Res = WidenVSELECTAndMask(N))
3944 return Res;
3945
3946 EVT CondEltVT = CondVT.getVectorElementType();
3947 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3948 CondEltVT, WidenNumElts);
3949 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3950 Cond1 = GetWidenedVector(Cond1);
3951
3952 // If we have to split the condition there is no point in widening the
3953 // select. This would result in an cycle of widening the select ->
3954 // widening the condition operand -> splitting the condition operand ->
3955 // splitting the select -> widening the select. Instead split this select
3956 // further and widen the resulting type.
3957 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3958 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3959 SDValue Res = ModifyToType(SplitSelect, WidenVT);
3960 return Res;
3961 }
3962
3963 if (Cond1.getValueType() != CondWidenVT)
3964 Cond1 = ModifyToType(Cond1, CondWidenVT);
3965 }
3966
3967 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3968 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3969 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3970 return DAG.getNode(N->getOpcode(), SDLoc(N),
3971 WidenVT, Cond1, InOp1, InOp2);
3972 }
3973
3974 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3975 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3976 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3977 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3978 InOp1.getValueType(), N->getOperand(0),
3979 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3980 }
3981
3982 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3983 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3984 return DAG.getUNDEF(WidenVT);
3985 }
3986
3987 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3988 EVT VT = N->getValueType(0);
3989 SDLoc dl(N);
3990
3991 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3992 unsigned NumElts = VT.getVectorNumElements();
3993 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3994
3995 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3996 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3997
3998 // Adjust mask based on new input vector length.
3999 SmallVector<int, 16> NewMask;
4000 for (unsigned i = 0; i != NumElts; ++i) {
4001 int Idx = N->getMaskElt(i);
4002 if (Idx < (int)NumElts)
4003 NewMask.push_back(Idx);
4004 else
4005 NewMask.push_back(Idx - NumElts + WidenNumElts);
4006 }
4007 for (unsigned i = NumElts; i != WidenNumElts; ++i)
4008 NewMask.push_back(-1);
4009 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4010 }
4011
4012 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4013 assert(N->getValueType(0).isVector() &&
4014 N->getOperand(0).getValueType().isVector() &&
4015 "Operands must be vectors");
4016 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4017 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4018
4019 SDValue InOp1 = N->getOperand(0);
4020 EVT InVT = InOp1.getValueType();
4021 assert(InVT.isVector() && "can not widen non-vector type");
4022 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4023 InVT.getVectorElementType(), WidenNumElts);
4024
4025 // The input and output types often differ here, and it could be that while
4026 // we'd prefer to widen the result type, the input operands have been split.
4027 // In this case, we also need to split the result of this node as well.
4028 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4029 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4030 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4031 return Res;
4032 }
4033
4034 // If the inputs also widen, handle them directly. Otherwise widen by hand.
4035 SDValue InOp2 = N->getOperand(1);
4036 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4037 InOp1 = GetWidenedVector(InOp1);
4038 InOp2 = GetWidenedVector(InOp2);
4039 } else {
4040 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4041 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4042 }
4043
4044 // Assume that the input and output will be widen appropriately. If not,
4045 // we will have to unroll it at some point.
4046 assert(InOp1.getValueType() == WidenInVT &&
4047 InOp2.getValueType() == WidenInVT &&
4048 "Input not widened to expected type!");
4049 (void)WidenInVT;
4050 return DAG.getNode(ISD::SETCC, SDLoc(N),
4051 WidenVT, InOp1, InOp2, N->getOperand(2));
4052 }
4053
4054
4055 //===----------------------------------------------------------------------===//
4056 // Widen Vector Operand
4057 //===----------------------------------------------------------------------===//
4058 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4059 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4060 dbgs() << "\n");
4061 SDValue Res = SDValue();
4062
4063 // See if the target wants to custom widen this node.
4064 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4065 return false;
4066
4067 switch (N->getOpcode()) {
4068 default:
4069 #ifndef NDEBUG
4070 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4071 N->dump(&DAG);
4072 dbgs() << "\n";
4073 #endif
4074 llvm_unreachable("Do not know how to widen this operator's operand!");
4075
4076 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4077 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4078 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4079 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4080 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4081 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4082 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4083 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4084 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4085 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4086 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4087
4088 case ISD::ANY_EXTEND:
4089 case ISD::SIGN_EXTEND:
4090 case ISD::ZERO_EXTEND:
4091 Res = WidenVecOp_EXTEND(N);
4092 break;
4093
4094 case ISD::FP_EXTEND:
4095 case ISD::STRICT_FP_EXTEND:
4096 case ISD::FP_TO_SINT:
4097 case ISD::FP_TO_UINT:
4098 case ISD::SINT_TO_FP:
4099 case ISD::UINT_TO_FP:
4100 case ISD::TRUNCATE:
4101 Res = WidenVecOp_Convert(N);
4102 break;
4103
4104 case ISD::VECREDUCE_FADD:
4105 case ISD::VECREDUCE_FMUL:
4106 case ISD::VECREDUCE_ADD:
4107 case ISD::VECREDUCE_MUL:
4108 case ISD::VECREDUCE_AND:
4109 case ISD::VECREDUCE_OR:
4110 case ISD::VECREDUCE_XOR:
4111 case ISD::VECREDUCE_SMAX:
4112 case ISD::VECREDUCE_SMIN:
4113 case ISD::VECREDUCE_UMAX:
4114 case ISD::VECREDUCE_UMIN:
4115 case ISD::VECREDUCE_FMAX:
4116 case ISD::VECREDUCE_FMIN:
4117 Res = WidenVecOp_VECREDUCE(N);
4118 break;
4119 }
4120
4121 // If Res is null, the sub-method took care of registering the result.
4122 if (!Res.getNode()) return false;
4123
4124 // If the result is N, the sub-method updated N in place. Tell the legalizer
4125 // core about this.
4126 if (Res.getNode() == N)
4127 return true;
4128
4129
4130 if (N->isStrictFPOpcode())
4131 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4132 "Invalid operand expansion");
4133 else
4134 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4135 "Invalid operand expansion");
4136
4137 ReplaceValueWith(SDValue(N, 0), Res);
4138 return false;
4139 }
4140
4141 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4142 SDLoc DL(N);
4143 EVT VT = N->getValueType(0);
4144
4145 SDValue InOp = N->getOperand(0);
4146 assert(getTypeAction(InOp.getValueType()) ==
4147 TargetLowering::TypeWidenVector &&
4148 "Unexpected type action");
4149 InOp = GetWidenedVector(InOp);
4150 assert(VT.getVectorNumElements() <
4151 InOp.getValueType().getVectorNumElements() &&
4152 "Input wasn't widened!");
4153
4154 // We may need to further widen the operand until it has the same total
4155 // vector size as the result.
4156 EVT InVT = InOp.getValueType();
4157 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4158 EVT InEltVT = InVT.getVectorElementType();
4159 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
4160 EVT FixedVT = (MVT::SimpleValueType)i;
4161 EVT FixedEltVT = FixedVT.getVectorElementType();
4162 if (TLI.isTypeLegal(FixedVT) &&
4163 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4164 FixedEltVT == InEltVT) {
4165 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4166 "Not enough elements in the fixed type for the operand!");
4167 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4168 "We can't have the same type as we started with!");
4169 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4170 InOp = DAG.getNode(
4171 ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
4172 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4173 else
4174 InOp = DAG.getNode(
4175 ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4176 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4177 break;
4178 }
4179 }
4180 InVT = InOp.getValueType();
4181 if (InVT.getSizeInBits() != VT.getSizeInBits())
4182 // We couldn't find a legal vector type that was a widening of the input
4183 // and could be extended in-register to the result type, so we have to
4184 // scalarize.
4185 return WidenVecOp_Convert(N);
4186 }
4187
4188 // Use special DAG nodes to represent the operation of extending the
4189 // low lanes.
4190 switch (N->getOpcode()) {
4191 default:
4192 llvm_unreachable("Extend legalization on extend operation!");
4193 case ISD::ANY_EXTEND:
4194 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4195 case ISD::SIGN_EXTEND:
4196 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4197 case ISD::ZERO_EXTEND:
4198 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4199 }
4200 }
4201
4202 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4203 // The result (and first input) is legal, but the second input is illegal.
4204 // We can't do much to fix that, so just unroll and let the extracts off of
4205 // the second input be widened as needed later.
4206 return DAG.UnrollVectorOp(N);
4207 }
4208
4209 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4210 // Since the result is legal and the input is illegal.
4211 EVT VT = N->getValueType(0);
4212 EVT EltVT = VT.getVectorElementType();
4213 SDLoc dl(N);
4214 unsigned NumElts = VT.getVectorNumElements();
4215 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4216 assert(getTypeAction(InOp.getValueType()) ==
4217 TargetLowering::TypeWidenVector &&
4218 "Unexpected type action");
4219 InOp = GetWidenedVector(InOp);
4220 EVT InVT = InOp.getValueType();
4221 unsigned Opcode = N->getOpcode();
4222
4223 // See if a widened result type would be legal, if so widen the node.
4224 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4225 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4226 InVT.getVectorNumElements());
4227 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4228 SDValue Res;
4229 if (N->isStrictFPOpcode()) {
4230 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4231 { N->getOperand(0), InOp });
4232 // Legalize the chain result - switch anything that used the old chain to
4233 // use the new one.
4234 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4235 } else
4236 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4237 return DAG.getNode(
4238 ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4239 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4240 }
4241
4242 EVT InEltVT = InVT.getVectorElementType();
4243
4244 // Unroll the convert into some scalar code and create a nasty build vector.
4245 SmallVector<SDValue, 16> Ops(NumElts);
4246 if (N->isStrictFPOpcode()) {
4247 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4248 SmallVector<SDValue, 32> OpChains;
4249 for (unsigned i=0; i < NumElts; ++i) {
4250 NewOps[1] = DAG.getNode(
4251 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4252 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4253 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4254 OpChains.push_back(Ops[i].getValue(1));
4255 }
4256 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4257 ReplaceValueWith(SDValue(N, 1), NewChain);
4258 } else {
4259 for (unsigned i = 0; i < NumElts; ++i)
4260 Ops[i] = DAG.getNode(
4261 Opcode, dl, EltVT,
4262 DAG.getNode(
4263 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4264 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4265 }
4266
4267 return DAG.getBuildVector(VT, dl, Ops);
4268 }
4269
4270 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4271 EVT VT = N->getValueType(0);
4272 SDValue InOp = GetWidenedVector(N->getOperand(0));
4273 EVT InWidenVT = InOp.getValueType();
4274 SDLoc dl(N);
4275
4276 // Check if we can convert between two legal vector types and extract.
4277 unsigned InWidenSize = InWidenVT.getSizeInBits();
4278 unsigned Size = VT.getSizeInBits();
4279 // x86mmx is not an acceptable vector element type, so don't try.
4280 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4281 unsigned NewNumElts = InWidenSize / Size;
4282 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4283 if (TLI.isTypeLegal(NewVT)) {
4284 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4285 return DAG.getNode(
4286 ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4287 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4288 }
4289 }
4290
4291 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4292 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4293 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4294 // having to copy via memory.
4295 if (VT.isVector()) {
4296 EVT EltVT = VT.getVectorElementType();
4297 unsigned EltSize = EltVT.getSizeInBits();
4298 if (InWidenSize % EltSize == 0) {
4299 unsigned NewNumElts = InWidenSize / EltSize;
4300 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4301 if (TLI.isTypeLegal(NewVT)) {
4302 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4303 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4304 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4305 }
4306 }
4307 }
4308
4309 return CreateStackStoreLoad(InOp, VT);
4310 }
4311
4312 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4313 EVT VT = N->getValueType(0);
4314 EVT EltVT = VT.getVectorElementType();
4315 EVT InVT = N->getOperand(0).getValueType();
4316 SDLoc dl(N);
4317
4318 // If the widen width for this operand is the same as the width of the concat
4319 // and all but the first operand is undef, just use the widened operand.
4320 unsigned NumOperands = N->getNumOperands();
4321 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4322 unsigned i;
4323 for (i = 1; i < NumOperands; ++i)
4324 if (!N->getOperand(i).isUndef())
4325 break;
4326
4327 if (i == NumOperands)
4328 return GetWidenedVector(N->getOperand(0));
4329 }
4330
4331 // Otherwise, fall back to a nasty build vector.
4332 unsigned NumElts = VT.getVectorNumElements();
4333 SmallVector<SDValue, 16> Ops(NumElts);
4334
4335 unsigned NumInElts = InVT.getVectorNumElements();
4336
4337 unsigned Idx = 0;
4338 for (unsigned i=0; i < NumOperands; ++i) {
4339 SDValue InOp = N->getOperand(i);
4340 assert(getTypeAction(InOp.getValueType()) ==
4341 TargetLowering::TypeWidenVector &&
4342 "Unexpected type action");
4343 InOp = GetWidenedVector(InOp);
4344 for (unsigned j=0; j < NumInElts; ++j)
4345 Ops[Idx++] = DAG.getNode(
4346 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4347 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4348 }
4349 return DAG.getBuildVector(VT, dl, Ops);
4350 }
4351
4352 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4353 SDValue InOp = GetWidenedVector(N->getOperand(0));
4354 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4355 N->getValueType(0), InOp, N->getOperand(1));
4356 }
4357
4358 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4359 SDValue InOp = GetWidenedVector(N->getOperand(0));
4360 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4361 N->getValueType(0), InOp, N->getOperand(1));
4362 }
4363
4364 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4365 // We have to widen the value, but we want only to store the original
4366 // vector type.
4367 StoreSDNode *ST = cast<StoreSDNode>(N);
4368
4369 if (!ST->getMemoryVT().getScalarType().isByteSized())
4370 return TLI.scalarizeVectorStore(ST, DAG);
4371
4372 SmallVector<SDValue, 16> StChain;
4373 if (ST->isTruncatingStore())
4374 GenWidenVectorTruncStores(StChain, ST);
4375 else
4376 GenWidenVectorStores(StChain, ST);
4377
4378 if (StChain.size() == 1)
4379 return StChain[0];
4380 else
4381 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4382 }
4383
4384 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4385 assert((OpNo == 1 || OpNo == 3) &&
4386 "Can widen only data or mask operand of mstore");
4387 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4388 SDValue Mask = MST->getMask();
4389 EVT MaskVT = Mask.getValueType();
4390 SDValue StVal = MST->getValue();
4391 SDLoc dl(N);
4392
4393 if (OpNo == 1) {
4394 // Widen the value.
4395 StVal = GetWidenedVector(StVal);
4396
4397 // The mask should be widened as well.
4398 EVT WideVT = StVal.getValueType();
4399 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4400 MaskVT.getVectorElementType(),
4401 WideVT.getVectorNumElements());
4402 Mask = ModifyToType(Mask, WideMaskVT, true);
4403 } else {
4404 // Widen the mask.
4405 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4406 Mask = ModifyToType(Mask, WideMaskVT, true);
4407
4408 EVT ValueVT = StVal.getValueType();
4409 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4410 ValueVT.getVectorElementType(),
4411 WideMaskVT.getVectorNumElements());
4412 StVal = ModifyToType(StVal, WideVT);
4413 }
4414
4415 assert(Mask.getValueType().getVectorNumElements() ==
4416 StVal.getValueType().getVectorNumElements() &&
4417 "Mask and data vectors should have the same number of elements");
4418 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4419 Mask, MST->getMemoryVT(), MST->getMemOperand(),
4420 false, MST->isCompressingStore());
4421 }
4422
4423 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4424 assert(OpNo == 4 && "Can widen only the index of mgather");
4425 auto *MG = cast<MaskedGatherSDNode>(N);
4426 SDValue DataOp = MG->getPassThru();
4427 SDValue Mask = MG->getMask();
4428 SDValue Scale = MG->getScale();
4429
4430 // Just widen the index. It's allowed to have extra elements.
4431 SDValue Index = GetWidenedVector(MG->getIndex());
4432
4433 SDLoc dl(N);
4434 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4435 Scale};
4436 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4437 MG->getMemOperand());
4438 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4439 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4440 return SDValue();
4441 }
4442
4443 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4444 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4445 SDValue DataOp = MSC->getValue();
4446 SDValue Mask = MSC->getMask();
4447 SDValue Index = MSC->getIndex();
4448 SDValue Scale = MSC->getScale();
4449
4450 if (OpNo == 1) {
4451 DataOp = GetWidenedVector(DataOp);
4452 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4453
4454 // Widen index.
4455 EVT IndexVT = Index.getValueType();
4456 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4457 IndexVT.getVectorElementType(), NumElts);
4458 Index = ModifyToType(Index, WideIndexVT);
4459
4460 // The mask should be widened as well.
4461 EVT MaskVT = Mask.getValueType();
4462 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4463 MaskVT.getVectorElementType(), NumElts);
4464 Mask = ModifyToType(Mask, WideMaskVT, true);
4465 } else if (OpNo == 4) {
4466 // Just widen the index. It's allowed to have extra elements.
4467 Index = GetWidenedVector(Index);
4468 } else
4469 llvm_unreachable("Can't widen this operand of mscatter");
4470
4471 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4472 Scale};
4473 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4474 MSC->getMemoryVT(), SDLoc(N), Ops,
4475 MSC->getMemOperand());
4476 }
4477
4478 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4479 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4480 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4481 SDLoc dl(N);
4482 EVT VT = N->getValueType(0);
4483
4484 // WARNING: In this code we widen the compare instruction with garbage.
4485 // This garbage may contain denormal floats which may be slow. Is this a real
4486 // concern ? Should we zero the unused lanes if this is a float compare ?
4487
4488 // Get a new SETCC node to compare the newly widened operands.
4489 // Only some of the compared elements are legal.
4490 EVT SVT = getSetCCResultType(InOp0.getValueType());
4491 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4492 if (VT.getScalarType() == MVT::i1)
4493 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4494 SVT.getVectorNumElements());
4495
4496 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4497 SVT, InOp0, InOp1, N->getOperand(2));
4498
4499 // Extract the needed results from the result vector.
4500 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4501 SVT.getVectorElementType(),
4502 VT.getVectorNumElements());
4503 SDValue CC = DAG.getNode(
4504 ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4505 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4506
4507 return PromoteTargetBoolean(CC, VT);
4508 }
4509
4510 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4511 SDLoc dl(N);
4512 SDValue Op = GetWidenedVector(N->getOperand(0));
4513 EVT OrigVT = N->getOperand(0).getValueType();
4514 EVT WideVT = Op.getValueType();
4515 EVT ElemVT = OrigVT.getVectorElementType();
4516
4517 SDValue NeutralElem;
4518 switch (N->getOpcode()) {
4519 case ISD::VECREDUCE_ADD:
4520 case ISD::VECREDUCE_OR:
4521 case ISD::VECREDUCE_XOR:
4522 case ISD::VECREDUCE_UMAX:
4523 NeutralElem = DAG.getConstant(0, dl, ElemVT);
4524 break;
4525 case ISD::VECREDUCE_MUL:
4526 NeutralElem = DAG.getConstant(1, dl, ElemVT);
4527 break;
4528 case ISD::VECREDUCE_AND:
4529 case ISD::VECREDUCE_UMIN:
4530 NeutralElem = DAG.getAllOnesConstant(dl, ElemVT);
4531 break;
4532 case ISD::VECREDUCE_SMAX:
4533 NeutralElem = DAG.getConstant(
4534 APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT);
4535 break;
4536 case ISD::VECREDUCE_SMIN:
4537 NeutralElem = DAG.getConstant(
4538 APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT);
4539 break;
4540 case ISD::VECREDUCE_FADD:
4541 NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT);
4542 break;
4543 case ISD::VECREDUCE_FMUL:
4544 NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT);
4545 break;
4546 case ISD::VECREDUCE_FMAX:
4547 NeutralElem = DAG.getConstantFP(
4548 std::numeric_limits<double>::infinity(), dl, ElemVT);
4549 break;
4550 case ISD::VECREDUCE_FMIN:
4551 NeutralElem = DAG.getConstantFP(
4552 -std::numeric_limits<double>::infinity(), dl, ElemVT);
4553 break;
4554 }
4555
4556 // Pad the vector with the neutral element.
4557 unsigned OrigElts = OrigVT.getVectorNumElements();
4558 unsigned WideElts = WideVT.getVectorNumElements();
4559 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
4560 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4561 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4562
4563 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags());
4564 }
4565
4566 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4567 // This only gets called in the case that the left and right inputs and
4568 // result are of a legal odd vector type, and the condition is illegal i1 of
4569 // the same odd width that needs widening.
4570 EVT VT = N->getValueType(0);
4571 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
4572
4573 SDValue Cond = GetWidenedVector(N->getOperand(0));
4574 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4575 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4576 SDLoc DL(N);
4577
4578 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4579 LeftIn, RightIn);
4580 return DAG.getNode(
4581 ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
4582 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4583 }
4584
4585 //===----------------------------------------------------------------------===//
4586 // Vector Widening Utilities
4587 //===----------------------------------------------------------------------===//
4588
4589 // Utility function to find the type to chop up a widen vector for load/store
4590 // TLI: Target lowering used to determine legal types.
4591 // Width: Width left need to load/store.
4592 // WidenVT: The widen vector type to load to/store from
4593 // Align: If 0, don't allow use of a wider type
4594 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4595
4596 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4597 unsigned Width, EVT WidenVT,
4598 unsigned Align = 0, unsigned WidenEx = 0) {
4599 EVT WidenEltVT = WidenVT.getVectorElementType();
4600 unsigned WidenWidth = WidenVT.getSizeInBits();
4601 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4602 unsigned AlignInBits = Align*8;
4603
4604 // If we have one element to load/store, return it.
4605 EVT RetVT = WidenEltVT;
4606 if (Width == WidenEltWidth)
4607 return RetVT;
4608
4609 // See if there is larger legal integer than the element type to load/store.
4610 unsigned VT;
4611 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4612 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4613 EVT MemVT((MVT::SimpleValueType) VT);
4614 unsigned MemVTWidth = MemVT.getSizeInBits();
4615 if (MemVT.getSizeInBits() <= WidenEltWidth)
4616 break;
4617 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4618 if ((Action == TargetLowering::TypeLegal ||
4619 Action == TargetLowering::TypePromoteInteger) &&
4620 (WidenWidth % MemVTWidth) == 0 &&
4621 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4622 (MemVTWidth <= Width ||
4623 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4624 if (MemVTWidth == WidenWidth)
4625 return MemVT;
4626 RetVT = MemVT;
4627 break;
4628 }
4629 }
4630
4631 // See if there is a larger vector type to load/store that has the same vector
4632 // element type and is evenly divisible with the WidenVT.
4633 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4634 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4635 EVT MemVT = (MVT::SimpleValueType) VT;
4636 unsigned MemVTWidth = MemVT.getSizeInBits();
4637 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4638 if ((Action == TargetLowering::TypeLegal ||
4639 Action == TargetLowering::TypePromoteInteger) &&
4640 WidenEltVT == MemVT.getVectorElementType() &&
4641 (WidenWidth % MemVTWidth) == 0 &&
4642 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4643 (MemVTWidth <= Width ||
4644 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4645 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4646 return MemVT;
4647 }
4648 }
4649
4650 return RetVT;
4651 }
4652
4653 // Builds a vector type from scalar loads
4654 // VecTy: Resulting Vector type
4655 // LDOps: Load operators to build a vector type
4656 // [Start,End) the list of loads to use.
4657 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4658 SmallVectorImpl<SDValue> &LdOps,
4659 unsigned Start, unsigned End) {
4660 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4661 SDLoc dl(LdOps[Start]);
4662 EVT LdTy = LdOps[Start].getValueType();
4663 unsigned Width = VecTy.getSizeInBits();
4664 unsigned NumElts = Width / LdTy.getSizeInBits();
4665 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4666
4667 unsigned Idx = 1;
4668 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4669
4670 for (unsigned i = Start + 1; i != End; ++i) {
4671 EVT NewLdTy = LdOps[i].getValueType();
4672 if (NewLdTy != LdTy) {
4673 NumElts = Width / NewLdTy.getSizeInBits();
4674 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4675 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4676 // Readjust position and vector position based on new load type.
4677 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4678 LdTy = NewLdTy;
4679 }
4680 VecOp = DAG.getNode(
4681 ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4682 DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4683 }
4684 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4685 }
4686
4687 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4688 LoadSDNode *LD) {
4689 // The strategy assumes that we can efficiently load power-of-two widths.
4690 // The routine chops the vector into the largest vector loads with the same
4691 // element type or scalar loads and then recombines it to the widen vector
4692 // type.
4693 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4694 unsigned WidenWidth = WidenVT.getSizeInBits();
4695 EVT LdVT = LD->getMemoryVT();
4696 SDLoc dl(LD);
4697 assert(LdVT.isVector() && WidenVT.isVector());
4698 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4699
4700 // Load information
4701 SDValue Chain = LD->getChain();
4702 SDValue BasePtr = LD->getBasePtr();
4703 unsigned Align = LD->getAlignment();
4704 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4705 AAMDNodes AAInfo = LD->getAAInfo();
4706
4707 int LdWidth = LdVT.getSizeInBits();
4708 int WidthDiff = WidenWidth - LdWidth;
4709 unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
4710
4711 // Find the vector type that can load from.
4712 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4713 int NewVTWidth = NewVT.getSizeInBits();
4714 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4715 Align, MMOFlags, AAInfo);
4716 LdChain.push_back(LdOp.getValue(1));
4717
4718 // Check if we can load the element with one instruction.
4719 if (LdWidth <= NewVTWidth) {
4720 if (!NewVT.isVector()) {
4721 unsigned NumElts = WidenWidth / NewVTWidth;
4722 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4723 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4724 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4725 }
4726 if (NewVT == WidenVT)
4727 return LdOp;
4728
4729 assert(WidenWidth % NewVTWidth == 0);
4730 unsigned NumConcat = WidenWidth / NewVTWidth;
4731 SmallVector<SDValue, 16> ConcatOps(NumConcat);
4732 SDValue UndefVal = DAG.getUNDEF(NewVT);
4733 ConcatOps[0] = LdOp;
4734 for (unsigned i = 1; i != NumConcat; ++i)
4735 ConcatOps[i] = UndefVal;
4736 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4737 }
4738
4739 // Load vector by using multiple loads from largest vector to scalar.
4740 SmallVector<SDValue, 16> LdOps;
4741 LdOps.push_back(LdOp);
4742
4743 LdWidth -= NewVTWidth;
4744 unsigned Offset = 0;
4745
4746 while (LdWidth > 0) {
4747 unsigned Increment = NewVTWidth / 8;
4748 Offset += Increment;
4749 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4750
4751 SDValue L;
4752 if (LdWidth < NewVTWidth) {
4753 // The current type we are using is too large. Find a better size.
4754 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4755 NewVTWidth = NewVT.getSizeInBits();
4756 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4757 LD->getPointerInfo().getWithOffset(Offset),
4758 MinAlign(Align, Increment), MMOFlags, AAInfo);
4759 LdChain.push_back(L.getValue(1));
4760 if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4761 // Later code assumes the vector loads produced will be mergeable, so we
4762 // must pad the final entry up to the previous width. Scalars are
4763 // combined separately.
4764 SmallVector<SDValue, 16> Loads;
4765 Loads.push_back(L);
4766 unsigned size = L->getValueSizeInBits(0);
4767 while (size < LdOp->getValueSizeInBits(0)) {
4768 Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4769 size += L->getValueSizeInBits(0);
4770 }
4771 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4772 }
4773 } else {
4774 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4775 LD->getPointerInfo().getWithOffset(Offset),
4776 MinAlign(Align, Increment), MMOFlags, AAInfo);
4777 LdChain.push_back(L.getValue(1));
4778 }
4779
4780 LdOps.push_back(L);
4781 LdOp = L;
4782
4783 LdWidth -= NewVTWidth;
4784 }
4785
4786 // Build the vector from the load operations.
4787 unsigned End = LdOps.size();
4788 if (!LdOps[0].getValueType().isVector())
4789 // All the loads are scalar loads.
4790 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4791
4792 // If the load contains vectors, build the vector using concat vector.
4793 // All of the vectors used to load are power-of-2, and the scalar loads can be
4794 // combined to make a power-of-2 vector.
4795 SmallVector<SDValue, 16> ConcatOps(End);
4796 int i = End - 1;
4797 int Idx = End;
4798 EVT LdTy = LdOps[i].getValueType();
4799 // First, combine the scalar loads to a vector.
4800 if (!LdTy.isVector()) {
4801 for (--i; i >= 0; --i) {
4802 LdTy = LdOps[i].getValueType();
4803 if (LdTy.isVector())
4804 break;
4805 }
4806 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4807 }
4808 ConcatOps[--Idx] = LdOps[i];
4809 for (--i; i >= 0; --i) {
4810 EVT NewLdTy = LdOps[i].getValueType();
4811 if (NewLdTy != LdTy) {
4812 // Create a larger vector.
4813 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4814 makeArrayRef(&ConcatOps[Idx], End - Idx));
4815 Idx = End - 1;
4816 LdTy = NewLdTy;
4817 }
4818 ConcatOps[--Idx] = LdOps[i];
4819 }
4820
4821 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4822 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4823 makeArrayRef(&ConcatOps[Idx], End - Idx));
4824
4825 // We need to fill the rest with undefs to build the vector.
4826 unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4827 SmallVector<SDValue, 16> WidenOps(NumOps);
4828 SDValue UndefVal = DAG.getUNDEF(LdTy);
4829 {
4830 unsigned i = 0;
4831 for (; i != End-Idx; ++i)
4832 WidenOps[i] = ConcatOps[Idx+i];
4833 for (; i != NumOps; ++i)
4834 WidenOps[i] = UndefVal;
4835 }
4836 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4837 }
4838
4839 SDValue
4840 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4841 LoadSDNode *LD,
4842 ISD::LoadExtType ExtType) {
4843 // For extension loads, it may not be more efficient to chop up the vector
4844 // and then extend it. Instead, we unroll the load and build a new vector.
4845 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4846 EVT LdVT = LD->getMemoryVT();
4847 SDLoc dl(LD);
4848 assert(LdVT.isVector() && WidenVT.isVector());
4849
4850 // Load information
4851 SDValue Chain = LD->getChain();
4852 SDValue BasePtr = LD->getBasePtr();
4853 unsigned Align = LD->getAlignment();
4854 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4855 AAMDNodes AAInfo = LD->getAAInfo();
4856
4857 EVT EltVT = WidenVT.getVectorElementType();
4858 EVT LdEltVT = LdVT.getVectorElementType();
4859 unsigned NumElts = LdVT.getVectorNumElements();
4860
4861 // Load each element and widen.
4862 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4863 SmallVector<SDValue, 16> Ops(WidenNumElts);
4864 unsigned Increment = LdEltVT.getSizeInBits() / 8;
4865 Ops[0] =
4866 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4867 LdEltVT, Align, MMOFlags, AAInfo);
4868 LdChain.push_back(Ops[0].getValue(1));
4869 unsigned i = 0, Offset = Increment;
4870 for (i=1; i < NumElts; ++i, Offset += Increment) {
4871 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4872 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4873 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
4874 Align, MMOFlags, AAInfo);
4875 LdChain.push_back(Ops[i].getValue(1));
4876 }
4877
4878 // Fill the rest with undefs.
4879 SDValue UndefVal = DAG.getUNDEF(EltVT);
4880 for (; i != WidenNumElts; ++i)
4881 Ops[i] = UndefVal;
4882
4883 return DAG.getBuildVector(WidenVT, dl, Ops);
4884 }
4885
4886 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4887 StoreSDNode *ST) {
4888 // The strategy assumes that we can efficiently store power-of-two widths.
4889 // The routine chops the vector into the largest vector stores with the same
4890 // element type or scalar stores.
4891 SDValue Chain = ST->getChain();
4892 SDValue BasePtr = ST->getBasePtr();
4893 unsigned Align = ST->getAlignment();
4894 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4895 AAMDNodes AAInfo = ST->getAAInfo();
4896 SDValue ValOp = GetWidenedVector(ST->getValue());
4897 SDLoc dl(ST);
4898
4899 EVT StVT = ST->getMemoryVT();
4900 unsigned StWidth = StVT.getSizeInBits();
4901 EVT ValVT = ValOp.getValueType();
4902 unsigned ValWidth = ValVT.getSizeInBits();
4903 EVT ValEltVT = ValVT.getVectorElementType();
4904 unsigned ValEltWidth = ValEltVT.getSizeInBits();
4905 assert(StVT.getVectorElementType() == ValEltVT);
4906
4907 int Idx = 0; // current index to store
4908 unsigned Offset = 0; // offset from base to store
4909 while (StWidth != 0) {
4910 // Find the largest vector type we can store with.
4911 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4912 unsigned NewVTWidth = NewVT.getSizeInBits();
4913 unsigned Increment = NewVTWidth / 8;
4914 if (NewVT.isVector()) {
4915 unsigned NumVTElts = NewVT.getVectorNumElements();
4916 do {
4917 SDValue EOp = DAG.getNode(
4918 ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
4919 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4920 StChain.push_back(DAG.getStore(
4921 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4922 MinAlign(Align, Offset), MMOFlags, AAInfo));
4923 StWidth -= NewVTWidth;
4924 Offset += Increment;
4925 Idx += NumVTElts;
4926
4927 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4928 } while (StWidth != 0 && StWidth >= NewVTWidth);
4929 } else {
4930 // Cast the vector to the scalar type we can store.
4931 unsigned NumElts = ValWidth / NewVTWidth;
4932 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4933 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4934 // Readjust index position based on new vector type.
4935 Idx = Idx * ValEltWidth / NewVTWidth;
4936 do {
4937 SDValue EOp = DAG.getNode(
4938 ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4939 DAG.getConstant(Idx++, dl,
4940 TLI.getVectorIdxTy(DAG.getDataLayout())));
4941 StChain.push_back(DAG.getStore(
4942 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4943 MinAlign(Align, Offset), MMOFlags, AAInfo));
4944 StWidth -= NewVTWidth;
4945 Offset += Increment;
4946 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4947 } while (StWidth != 0 && StWidth >= NewVTWidth);
4948 // Restore index back to be relative to the original widen element type.
4949 Idx = Idx * NewVTWidth / ValEltWidth;
4950 }
4951 }
4952 }
4953
4954 void
4955 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4956 StoreSDNode *ST) {
4957 // For extension loads, it may not be more efficient to truncate the vector
4958 // and then store it. Instead, we extract each element and then store it.
4959 SDValue Chain = ST->getChain();
4960 SDValue BasePtr = ST->getBasePtr();
4961 unsigned Align = ST->getAlignment();
4962 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4963 AAMDNodes AAInfo = ST->getAAInfo();
4964 SDValue ValOp = GetWidenedVector(ST->getValue());
4965 SDLoc dl(ST);
4966
4967 EVT StVT = ST->getMemoryVT();
4968 EVT ValVT = ValOp.getValueType();
4969
4970 // It must be true that the wide vector type is bigger than where we need to
4971 // store.
4972 assert(StVT.isVector() && ValOp.getValueType().isVector());
4973 assert(StVT.bitsLT(ValOp.getValueType()));
4974
4975 // For truncating stores, we can not play the tricks of chopping legal vector
4976 // types and bitcast it to the right type. Instead, we unroll the store.
4977 EVT StEltVT = StVT.getVectorElementType();
4978 EVT ValEltVT = ValVT.getVectorElementType();
4979 unsigned Increment = ValEltVT.getSizeInBits() / 8;
4980 unsigned NumElts = StVT.getVectorNumElements();
4981 SDValue EOp = DAG.getNode(
4982 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4983 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4984 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4985 ST->getPointerInfo(), StEltVT, Align,
4986 MMOFlags, AAInfo));
4987 unsigned Offset = Increment;
4988 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4989 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4990 SDValue EOp = DAG.getNode(
4991 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4992 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4993 StChain.push_back(DAG.getTruncStore(
4994 Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4995 StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4996 }
4997 }
4998
4999 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
5000 /// input vector must have the same element type as NVT.
5001 /// FillWithZeroes specifies that the vector should be widened with zeroes.
5002 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5003 bool FillWithZeroes) {
5004 // Note that InOp might have been widened so it might already have
5005 // the right width or it might need be narrowed.
5006 EVT InVT = InOp.getValueType();
5007 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5008 "input and widen element type must match");
5009 SDLoc dl(InOp);
5010
5011 // Check if InOp already has the right width.
5012 if (InVT == NVT)
5013 return InOp;
5014
5015 unsigned InNumElts = InVT.getVectorNumElements();
5016 unsigned WidenNumElts = NVT.getVectorNumElements();
5017 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5018 unsigned NumConcat = WidenNumElts / InNumElts;
5019 SmallVector<SDValue, 16> Ops(NumConcat);
5020 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5021 DAG.getUNDEF(InVT);
5022 Ops[0] = InOp;
5023 for (unsigned i = 1; i != NumConcat; ++i)
5024 Ops[i] = FillVal;
5025
5026 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5027 }
5028
5029 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
5030 return DAG.getNode(
5031 ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5032 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5033
5034 // Fall back to extract and build.
5035 SmallVector<SDValue, 16> Ops(WidenNumElts);
5036 EVT EltVT = NVT.getVectorElementType();
5037 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5038 unsigned Idx;
5039 for (Idx = 0; Idx < MinNumElts; ++Idx)
5040 Ops[Idx] = DAG.getNode(
5041 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5042 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5043
5044 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5045 DAG.getUNDEF(EltVT);
5046 for ( ; Idx < WidenNumElts; ++Idx)
5047 Ops[Idx] = FillVal;
5048 return DAG.getBuildVector(NVT, dl, Ops);
5049 }
The low level virtual machine