mlir/lib/Interfaces/DataLayoutInterfaces.cpp

   1 //===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "mlir/Interfaces/DataLayoutInterfaces.h"
  10 #include "mlir/IR/BuiltinDialect.h"
  11 #include "mlir/IR/BuiltinOps.h"
  12 #include "mlir/IR/BuiltinTypes.h"
  13 #include "mlir/IR/Operation.h"
  14
  15 #include "llvm/ADT/TypeSwitch.h"
  16 #include "llvm/Support/MathExtras.h"
  17
  18 using namespace mlir;
  19
  20 //===----------------------------------------------------------------------===//
  21 // Default implementations
  22 //===----------------------------------------------------------------------===//
  23
  24 /// Reports that the given type is missing the data layout information and
  25 /// exits.
  26 [[noreturn]] static void reportMissingDataLayout(Type type) {
  27   std::string message;
  28   llvm::raw_string_ostream os(message);
  29   os << "neither the scoping op nor the type class provide data layout "
  30         "information for "
  31      << type;
  32   llvm::report_fatal_error(Twine(os.str()));
  33 }
  34
  35 /// Returns the bitwidth of the index type if specified in the param list.
  36 /// Assumes 64-bit index otherwise.
  37 static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
  38   if (params.empty())
  39     return 64;
  40   auto attr = cast<IntegerAttr>(params.front().getValue());
  41   return attr.getValue().getZExtValue();
  42 }
  43
  44 llvm::TypeSize
  45 mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
  46                                  ArrayRef<DataLayoutEntryInterface> params) {
  47   llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
  48   return divideCeil(bits, 8);
  49 }
  50
  51 llvm::TypeSize
  52 mlir::detail::getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,
  53                                        DataLayoutEntryListRef params) {
  54   if (isa<IntegerType, FloatType>(type))
  55     return llvm::TypeSize::getFixed(type.getIntOrFloatBitWidth());
  56
  57   if (auto ctype = dyn_cast<ComplexType>(type)) {
  58     Type et = ctype.getElementType();
  59     uint64_t innerAlignment =
  60         getDefaultPreferredAlignment(et, dataLayout, params) * 8;
  61     llvm::TypeSize innerSize = getDefaultTypeSizeInBits(et, dataLayout, params);
  62
  63     // Include padding required to align the imaginary value in the complex
  64     // type.
  65     return llvm::alignTo(innerSize, innerAlignment) + innerSize;
  66   }
  67
  68   // Index is an integer of some bitwidth.
  69   if (isa<IndexType>(type))
  70     return dataLayout.getTypeSizeInBits(
  71         IntegerType::get(type.getContext(), getIndexBitwidth(params)));
  72
  73   // Sizes of vector types are rounded up to those of types with closest
  74   // power-of-two number of elements in the innermost dimension. We also assume
  75   // there is no bit-packing at the moment element sizes are taken in bytes and
  76   // multiplied with 8 bits.
  77   // TODO: make this extensible.
  78   if (auto vecType = dyn_cast<VectorType>(type)) {
  79     uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
  80                         llvm::PowerOf2Ceil(vecType.getShape().back()) *
  81                         dataLayout.getTypeSize(vecType.getElementType()) * 8;
  82     return llvm::TypeSize::get(baseSize, vecType.isScalable());
  83   }
  84
  85   if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
  86     return typeInterface.getTypeSizeInBits(dataLayout, params);
  87
  88   reportMissingDataLayout(type);
  89 }
  90
  91 static DataLayoutEntryInterface
  92 findEntryForIntegerType(IntegerType intType,
  93                         ArrayRef<DataLayoutEntryInterface> params) {
  94   assert(!params.empty() && "expected non-empty parameter list");
  95   std::map<unsigned, DataLayoutEntryInterface> sortedParams;
  96   for (DataLayoutEntryInterface entry : params) {
  97     sortedParams.insert(std::make_pair(
  98         entry.getKey().get<Type>().getIntOrFloatBitWidth(), entry));
  99   }
 100   auto iter = sortedParams.lower_bound(intType.getWidth());
 101   if (iter == sortedParams.end())
 102     iter = std::prev(iter);
 103
 104   return iter->second;
 105 }
 106
 107 constexpr const static uint64_t kDefaultBitsInByte = 8u;
 108
 109 static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
 110   auto values =
 111       cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
 112   return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
 113 }
 114
 115 static uint64_t
 116 getIntegerTypeABIAlignment(IntegerType intType,
 117                            ArrayRef<DataLayoutEntryInterface> params) {
 118   constexpr uint64_t kDefaultSmallIntAlignment = 4u;
 119   constexpr unsigned kSmallIntSize = 64;
 120   if (params.empty()) {
 121     return intType.getWidth() < kSmallIntSize
 122                ? llvm::PowerOf2Ceil(
 123                      llvm::divideCeil(intType.getWidth(), kDefaultBitsInByte))
 124                : kDefaultSmallIntAlignment;
 125   }
 126
 127   return extractABIAlignment(findEntryForIntegerType(intType, params));
 128 }
 129
 130 static uint64_t
 131 getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
 132                          ArrayRef<DataLayoutEntryInterface> params) {
 133   assert(params.size() <= 1 && "at most one data layout entry is expected for "
 134                                "the singleton floating-point type");
 135   if (params.empty())
 136     return llvm::PowerOf2Ceil(dataLayout.getTypeSize(fltType).getFixedValue());
 137   return extractABIAlignment(params[0]);
 138 }
 139
 140 uint64_t mlir::detail::getDefaultABIAlignment(
 141     Type type, const DataLayout &dataLayout,
 142     ArrayRef<DataLayoutEntryInterface> params) {
 143   // Natural alignment is the closest power-of-two number above. For scalable
 144   // vectors, aligning them to the same as the base vector is sufficient.
 145   if (isa<VectorType>(type))
 146     return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type).getKnownMinValue());
 147
 148   if (auto fltType = dyn_cast<FloatType>(type))
 149     return getFloatTypeABIAlignment(fltType, dataLayout, params);
 150
 151   // Index is an integer of some bitwidth.
 152   if (isa<IndexType>(type))
 153     return dataLayout.getTypeABIAlignment(
 154         IntegerType::get(type.getContext(), getIndexBitwidth(params)));
 155
 156   if (auto intType = dyn_cast<IntegerType>(type))
 157     return getIntegerTypeABIAlignment(intType, params);
 158
 159   if (auto ctype = dyn_cast<ComplexType>(type))
 160     return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
 161
 162   if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
 163     return typeInterface.getABIAlignment(dataLayout, params);
 164
 165   reportMissingDataLayout(type);
 166 }
 167
 168 static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
 169   auto values =
 170       cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
 171   return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;
 172 }
 173
 174 static uint64_t
 175 getIntegerTypePreferredAlignment(IntegerType intType,
 176                                  const DataLayout &dataLayout,
 177                                  ArrayRef<DataLayoutEntryInterface> params) {
 178   if (params.empty())
 179     return llvm::PowerOf2Ceil(dataLayout.getTypeSize(intType).getFixedValue());
 180
 181   return extractPreferredAlignment(findEntryForIntegerType(intType, params));
 182 }
 183
 184 static uint64_t
 185 getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
 186                                ArrayRef<DataLayoutEntryInterface> params) {
 187   assert(params.size() <= 1 && "at most one data layout entry is expected for "
 188                                "the singleton floating-point type");
 189   if (params.empty())
 190     return dataLayout.getTypeABIAlignment(fltType);
 191   return extractPreferredAlignment(params[0]);
 192 }
 193
 194 uint64_t mlir::detail::getDefaultPreferredAlignment(
 195     Type type, const DataLayout &dataLayout,
 196     ArrayRef<DataLayoutEntryInterface> params) {
 197   // Preferred alignment is same as natural for floats and vectors.
 198   if (isa<VectorType>(type))
 199     return dataLayout.getTypeABIAlignment(type);
 200
 201   if (auto fltType = dyn_cast<FloatType>(type))
 202     return getFloatTypePreferredAlignment(fltType, dataLayout, params);
 203
 204   // Preferred alignment is the closest power-of-two number above for integers
 205   // (ABI alignment may be smaller).
 206   if (auto intType = dyn_cast<IntegerType>(type))
 207     return getIntegerTypePreferredAlignment(intType, dataLayout, params);
 208
 209   if (isa<IndexType>(type)) {
 210     return dataLayout.getTypePreferredAlignment(
 211         IntegerType::get(type.getContext(), getIndexBitwidth(params)));
 212   }
 213
 214   if (auto ctype = dyn_cast<ComplexType>(type))
 215     return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
 216                                         params);
 217
 218   if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
 219     return typeInterface.getPreferredAlignment(dataLayout, params);
 220
 221   reportMissingDataLayout(type);
 222 }
 223
 224 std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth(
 225     Type type, const DataLayout &dataLayout,
 226     ArrayRef<DataLayoutEntryInterface> params) {
 227   if (isa<IndexType>(type))
 228     return getIndexBitwidth(params);
 229
 230   if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
 231     if (std::optional<uint64_t> indexBitwidth =
 232             typeInterface.getIndexBitwidth(dataLayout, params))
 233       return *indexBitwidth;
 234
 235   // Return std::nullopt for all other types, which are assumed to be non
 236   // pointer-like types.
 237   return std::nullopt;
 238 }
 239
 240 // Returns the endianness if specified in the given entry. If the entry is empty
 241 // the default endianness represented by an empty attribute is returned.
 242 Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
 243   if (entry == DataLayoutEntryInterface())
 244     return Attribute();
 245
 246   return entry.getValue();
 247 }
 248
 249 // Returns the memory space used for alloca operations if specified in the
 250 // given entry. If the entry is empty the default memory space represented by
 251 // an empty attribute is returned.
 252 Attribute
 253 mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
 254   if (entry == DataLayoutEntryInterface()) {
 255     return Attribute();
 256   }
 257
 258   return entry.getValue();
 259 }
 260
 261 // Returns the memory space used for the program memory space.  if
 262 // specified in the given entry. If the entry is empty the default
 263 // memory space represented by an empty attribute is returned.
 264 Attribute
 265 mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
 266   if (entry == DataLayoutEntryInterface()) {
 267     return Attribute();
 268   }
 269
 270   return entry.getValue();
 271 }
 272
 273 // Returns the memory space used for global the global memory space. if
 274 // specified in the given entry. If the entry is empty the default memory
 275 // space represented by an empty attribute is returned.
 276 Attribute
 277 mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
 278   if (entry == DataLayoutEntryInterface()) {
 279     return Attribute();
 280   }
 281
 282   return entry.getValue();
 283 }
 284
 285 // Returns the stack alignment if specified in the given entry. If the entry is
 286 // empty the default alignment zero is returned.
 287 uint64_t
 288 mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
 289   if (entry == DataLayoutEntryInterface())
 290     return 0;
 291
 292   auto value = cast<IntegerAttr>(entry.getValue());
 293   return value.getValue().getZExtValue();
 294 }
 295
 296 DataLayoutEntryList
 297 mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,
 298                                    TypeID typeID) {
 299   return llvm::to_vector<4>(llvm::make_filter_range(
 300       entries, [typeID](DataLayoutEntryInterface entry) {
 301         auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
 302         return type && type.getTypeID() == typeID;
 303       }));
 304 }
 305
 306 DataLayoutEntryInterface
 307 mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
 308                                        StringAttr id) {
 309   const auto *it = llvm::find_if(entries, [id](DataLayoutEntryInterface entry) {
 310     if (!entry.getKey().is<StringAttr>())
 311       return false;
 312     return entry.getKey().get<StringAttr>() == id;
 313   });
 314   return it == entries.end() ? DataLayoutEntryInterface() : *it;
 315 }
 316
 317 static DataLayoutSpecInterface getSpec(Operation *operation) {
 318   return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
 319       .Case<ModuleOp, DataLayoutOpInterface>(
 320           [&](auto op) { return op.getDataLayoutSpec(); })
 321       .Default([](Operation *) {
 322         llvm_unreachable("expected an op with data layout spec");
 323         return DataLayoutSpecInterface();
 324       });
 325 }
 326
 327 /// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
 328 /// are either modules or implement the `DataLayoutOpInterface`.
 329 static void
 330 collectParentLayouts(Operation *leaf,
 331                      SmallVectorImpl<DataLayoutSpecInterface> &specs,
 332                      SmallVectorImpl<Location> *opLocations = nullptr) {
 333   if (!leaf)
 334     return;
 335
 336   for (Operation *parent = leaf->getParentOp(); parent != nullptr;
 337        parent = parent->getParentOp()) {
 338     llvm::TypeSwitch<Operation *>(parent)
 339         .Case<ModuleOp>([&](ModuleOp op) {
 340           // Skip top-level module op unless it has a layout. Top-level module
 341           // without layout is most likely the one implicitly added by the
 342           // parser and it doesn't have location. Top-level null specification
 343           // would have had the same effect as not having a specification at all
 344           // (using type defaults).
 345           if (!op->getParentOp() && !op.getDataLayoutSpec())
 346             return;
 347           specs.push_back(op.getDataLayoutSpec());
 348           if (opLocations)
 349             opLocations->push_back(op.getLoc());
 350         })
 351         .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
 352           specs.push_back(op.getDataLayoutSpec());
 353           if (opLocations)
 354             opLocations->push_back(op.getLoc());
 355         });
 356   }
 357 }
 358
 359 /// Returns a layout spec that is a combination of the layout specs attached
 360 /// to the given operation and all its ancestors.
 361 static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
 362   if (!leaf)
 363     return {};
 364
 365   assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
 366          "expected an op with data layout spec");
 367
 368   SmallVector<DataLayoutOpInterface> opsWithLayout;
 369   SmallVector<DataLayoutSpecInterface> specs;
 370   collectParentLayouts(leaf, specs);
 371
 372   // Fast track if there are no ancestors.
 373   if (specs.empty())
 374     return getSpec(leaf);
 375
 376   // Create the list of non-null specs (null/missing specs can be safely
 377   // ignored) from the outermost to the innermost.
 378   auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
 379       llvm::reverse(specs),
 380       [](DataLayoutSpecInterface iface) { return iface != nullptr; }));
 381
 382   // Combine the specs using the innermost as anchor.
 383   if (DataLayoutSpecInterface current = getSpec(leaf))
 384     return current.combineWith(nonNullSpecs);
 385   if (nonNullSpecs.empty())
 386     return {};
 387   return nonNullSpecs.back().combineWith(
 388       llvm::ArrayRef(nonNullSpecs).drop_back());
 389 }
 390
 391 LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
 392   DataLayoutSpecInterface spec = getSpec(op);
 393   // The layout specification may be missing and it's fine.
 394   if (!spec)
 395     return success();
 396
 397   if (failed(spec.verifySpec(op->getLoc())))
 398     return failure();
 399   if (!getCombinedDataLayout(op)) {
 400     InFlightDiagnostic diag =
 401         op->emitError()
 402         << "data layout does not combine with layouts of enclosing ops";
 403     SmallVector<DataLayoutSpecInterface> specs;
 404     SmallVector<Location> opLocations;
 405     collectParentLayouts(op, specs, &opLocations);
 406     for (Location loc : opLocations)
 407       diag.attachNote(loc) << "enclosing op with data layout";
 408     return diag;
 409   }
 410   return success();
 411 }
 412
 413 llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
 414                                         uint64_t denominator) {
 415   uint64_t divided =
 416       llvm::divideCeil(numerator.getKnownMinValue(), denominator);
 417   return llvm::TypeSize::get(divided, numerator.isScalable());
 418 }
 419
 420 //===----------------------------------------------------------------------===//
 421 // DataLayout
 422 //===----------------------------------------------------------------------===//
 423
 424 template <typename OpTy>
 425 void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
 426   if (!originalLayout) {
 427     assert((!op || !op.getDataLayoutSpec()) &&
 428            "could not compute layout information for an op (failed to "
 429            "combine attributes?)");
 430   }
 431 }
 432
 433 mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
 434
 435 mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
 436     : originalLayout(getCombinedDataLayout(op)), scope(op),
 437       allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
 438       globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
 439 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
 440   checkMissingLayout(originalLayout, op);
 441   collectParentLayouts(op, layoutStack);
 442 #endif
 443 }
 444
 445 mlir::DataLayout::DataLayout(ModuleOp op)
 446     : originalLayout(getCombinedDataLayout(op)), scope(op),
 447       allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
 448       globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
 449 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
 450   checkMissingLayout(originalLayout, op);
 451   collectParentLayouts(op, layoutStack);
 452 #endif
 453 }
 454
 455 mlir::DataLayout mlir::DataLayout::closest(Operation *op) {
 456   // Search the closest parent either being a module operation or implementing
 457   // the data layout interface.
 458   while (op) {
 459     if (auto module = dyn_cast<ModuleOp>(op))
 460       return DataLayout(module);
 461     if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
 462       return DataLayout(iface);
 463     op = op->getParentOp();
 464   }
 465   return DataLayout();
 466 }
 467
 468 void mlir::DataLayout::checkValid() const {
 469 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
 470   SmallVector<DataLayoutSpecInterface> specs;
 471   collectParentLayouts(scope, specs);
 472   assert(specs.size() == layoutStack.size() &&
 473          "data layout object used, but no longer valid due to the change in "
 474          "number of nested layouts");
 475   for (auto pair : llvm::zip(specs, layoutStack)) {
 476     Attribute newLayout = std::get<0>(pair);
 477     Attribute origLayout = std::get<1>(pair);
 478     assert(newLayout == origLayout &&
 479            "data layout object used, but no longer valid "
 480            "due to the change in layout attributes");
 481   }
 482 #endif
 483   assert(((!scope && !this->originalLayout) ||
 484           (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
 485          "data layout object used, but no longer valid due to the change in "
 486          "layout spec");
 487 }
 488
 489 /// Looks up the value for the given type key in the given cache. If there is no
 490 /// such value in the cache, compute it using the given callback and put it in
 491 /// the cache before returning.
 492 template <typename T>
 493 static T cachedLookup(Type t, DenseMap<Type, T> &cache,
 494                       function_ref<T(Type)> compute) {
 495   auto it = cache.find(t);
 496   if (it != cache.end())
 497     return it->second;
 498
 499   auto result = cache.try_emplace(t, compute(t));
 500   return result.first->second;
 501 }
 502
 503 llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
 504   checkValid();
 505   return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {
 506     DataLayoutEntryList list;
 507     if (originalLayout)
 508       list = originalLayout.getSpecForType(ty.getTypeID());
 509     if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 510       return iface.getTypeSize(ty, *this, list);
 511     return detail::getDefaultTypeSize(ty, *this, list);
 512   });
 513 }
 514
 515 llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
 516   checkValid();
 517   return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {
 518     DataLayoutEntryList list;
 519     if (originalLayout)
 520       list = originalLayout.getSpecForType(ty.getTypeID());
 521     if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 522       return iface.getTypeSizeInBits(ty, *this, list);
 523     return detail::getDefaultTypeSizeInBits(ty, *this, list);
 524   });
 525 }
 526
 527 uint64_t mlir::DataLayout::getTypeABIAlignment(Type t) const {
 528   checkValid();
 529   return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {
 530     DataLayoutEntryList list;
 531     if (originalLayout)
 532       list = originalLayout.getSpecForType(ty.getTypeID());
 533     if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 534       return iface.getTypeABIAlignment(ty, *this, list);
 535     return detail::getDefaultABIAlignment(ty, *this, list);
 536   });
 537 }
 538
 539 uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const {
 540   checkValid();
 541   return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {
 542     DataLayoutEntryList list;
 543     if (originalLayout)
 544       list = originalLayout.getSpecForType(ty.getTypeID());
 545     if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 546       return iface.getTypePreferredAlignment(ty, *this, list);
 547     return detail::getDefaultPreferredAlignment(ty, *this, list);
 548   });
 549 }
 550
 551 std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
 552   checkValid();
 553   return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {
 554     DataLayoutEntryList list;
 555     if (originalLayout)
 556       list = originalLayout.getSpecForType(ty.getTypeID());
 557     if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 558       return iface.getIndexBitwidth(ty, *this, list);
 559     return detail::getDefaultIndexBitwidth(ty, *this, list);
 560   });
 561 }
 562
 563 mlir::Attribute mlir::DataLayout::getEndianness() const {
 564   checkValid();
 565   if (endianness)
 566     return *endianness;
 567   DataLayoutEntryInterface entry;
 568   if (originalLayout)
 569     entry = originalLayout.getSpecForIdentifier(
 570         originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
 571
 572   if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 573     endianness = iface.getEndianness(entry);
 574   else
 575     endianness = detail::getDefaultEndianness(entry);
 576   return *endianness;
 577 }
 578
 579 mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {
 580   checkValid();
 581   if (allocaMemorySpace)
 582     return *allocaMemorySpace;
 583   DataLayoutEntryInterface entry;
 584   if (originalLayout)
 585     entry = originalLayout.getSpecForIdentifier(
 586         originalLayout.getAllocaMemorySpaceIdentifier(
 587             originalLayout.getContext()));
 588   if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 589     allocaMemorySpace = iface.getAllocaMemorySpace(entry);
 590   else
 591     allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry);
 592   return *allocaMemorySpace;
 593 }
 594
 595 mlir::Attribute mlir::DataLayout::getProgramMemorySpace() const {
 596   checkValid();
 597   if (programMemorySpace)
 598     return *programMemorySpace;
 599   DataLayoutEntryInterface entry;
 600   if (originalLayout)
 601     entry = originalLayout.getSpecForIdentifier(
 602         originalLayout.getProgramMemorySpaceIdentifier(
 603             originalLayout.getContext()));
 604   if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 605     programMemorySpace = iface.getProgramMemorySpace(entry);
 606   else
 607     programMemorySpace = detail::getDefaultProgramMemorySpace(entry);
 608   return *programMemorySpace;
 609 }
 610
 611 mlir::Attribute mlir::DataLayout::getGlobalMemorySpace() const {
 612   checkValid();
 613   if (globalMemorySpace)
 614     return *globalMemorySpace;
 615   DataLayoutEntryInterface entry;
 616   if (originalLayout)
 617     entry = originalLayout.getSpecForIdentifier(
 618         originalLayout.getGlobalMemorySpaceIdentifier(
 619             originalLayout.getContext()));
 620   if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 621     globalMemorySpace = iface.getGlobalMemorySpace(entry);
 622   else
 623     globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry);
 624   return *globalMemorySpace;
 625 }
 626
 627 uint64_t mlir::DataLayout::getStackAlignment() const {
 628   checkValid();
 629   if (stackAlignment)
 630     return *stackAlignment;
 631   DataLayoutEntryInterface entry;
 632   if (originalLayout)
 633     entry = originalLayout.getSpecForIdentifier(
 634         originalLayout.getStackAlignmentIdentifier(
 635             originalLayout.getContext()));
 636   if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
 637     stackAlignment = iface.getStackAlignment(entry);
 638   else
 639     stackAlignment = detail::getDefaultStackAlignment(entry);
 640   return *stackAlignment;
 641 }
 642
 643 //===----------------------------------------------------------------------===//
 644 // DataLayoutSpecInterface
 645 //===----------------------------------------------------------------------===//
 646
 647 void DataLayoutSpecInterface::bucketEntriesByType(
 648     DenseMap<TypeID, DataLayoutEntryList> &types,
 649     DenseMap<StringAttr, DataLayoutEntryInterface> &ids) {
 650   for (DataLayoutEntryInterface entry : getEntries()) {
 651     if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
 652       types[type.getTypeID()].push_back(entry);
 653     else
 654       ids[entry.getKey().get<StringAttr>()] = entry;
 655   }
 656 }
 657
 658 LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
 659                                                  Location loc) {
 660   // First, verify individual entries.
 661   for (DataLayoutEntryInterface entry : spec.getEntries())
 662     if (failed(entry.verifyEntry(loc)))
 663       return failure();
 664
 665   // Second, dispatch verifications of entry groups to types or dialects they
 666   // are associated with.
 667   DenseMap<TypeID, DataLayoutEntryList> types;
 668   DenseMap<StringAttr, DataLayoutEntryInterface> ids;
 669   spec.bucketEntriesByType(types, ids);
 670
 671   for (const auto &kvp : types) {
 672     auto sampleType = kvp.second.front().getKey().get<Type>();
 673     if (isa<IndexType>(sampleType)) {
 674       assert(kvp.second.size() == 1 &&
 675              "expected one data layout entry for non-parametric 'index' type");
 676       if (!isa<IntegerAttr>(kvp.second.front().getValue()))
 677         return emitError(loc)
 678                << "expected integer attribute in the data layout entry for "
 679                << sampleType;
 680       continue;
 681     }
 682
 683     if (isa<IntegerType, FloatType>(sampleType)) {
 684       for (DataLayoutEntryInterface entry : kvp.second) {
 685         auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
 686         if (!value || !value.getElementType().isSignlessInteger(64)) {
 687           emitError(loc) << "expected a dense i64 elements attribute in the "
 688                             "data layout entry "
 689                          << entry;
 690           return failure();
 691         }
 692
 693         auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
 694         unsigned numElements = elements.size();
 695         if (numElements < 1 || numElements > 2) {
 696           emitError(loc) << "expected 1 or 2 elements in the data layout entry "
 697                          << entry;
 698           return failure();
 699         }
 700
 701         uint64_t abi = elements[0];
 702         uint64_t preferred = numElements == 2 ? elements[1] : abi;
 703         if (preferred < abi) {
 704           emitError(loc)
 705               << "preferred alignment is expected to be greater than or equal "
 706                  "to the abi alignment in data layout entry "
 707               << entry;
 708           return failure();
 709         }
 710       }
 711       continue;
 712     }
 713
 714     if (isa<BuiltinDialect>(&sampleType.getDialect()))
 715       return emitError(loc) << "unexpected data layout for a built-in type";
 716
 717     auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
 718     if (!dlType)
 719       return emitError(loc)
 720              << "data layout specified for a type that does not support it";
 721     if (failed(dlType.verifyEntries(kvp.second, loc)))
 722       return failure();
 723   }
 724
 725   for (const auto &kvp : ids) {
 726     StringAttr identifier = kvp.second.getKey().get<StringAttr>();
 727     Dialect *dialect = identifier.getReferencedDialect();
 728
 729     // Ignore attributes that belong to an unknown dialect, the dialect may
 730     // actually implement the relevant interface but we don't know about that.
 731     if (!dialect)
 732       continue;
 733
 734     const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
 735     if (!iface) {
 736       return emitError(loc)
 737              << "the '" << dialect->getNamespace()
 738              << "' dialect does not support identifier data layout entries";
 739     }
 740     if (failed(iface->verifyEntry(kvp.second, loc)))
 741       return failure();
 742   }
 743
 744   return success();
 745 }
 746
 747 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
 748 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
 749 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"