/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ |* *| |* Op Definitions *| |* *| |* Automatically generated file, do not edit! *| |* From: NVGPUOps.td *| |* *| \*===----------------------------------------------------------------------===*/ #ifdef GET_OP_LIST #undef GET_OP_LIST ::mlir::nvgpu::DeviceAsyncCopyOp, ::mlir::nvgpu::DeviceAsyncCreateGroupOp, ::mlir::nvgpu::DeviceAsyncWaitOp, ::mlir::nvgpu::LdMatrixOp, ::mlir::nvgpu::MBarrierArriveExpectTxOp, ::mlir::nvgpu::MBarrierArriveNoCompleteOp, ::mlir::nvgpu::MBarrierArriveOp, ::mlir::nvgpu::MBarrierCreateOp, ::mlir::nvgpu::MBarrierGetOp, ::mlir::nvgpu::MBarrierInitOp, ::mlir::nvgpu::MBarrierTestWaitOp, ::mlir::nvgpu::MBarrierTryWaitParityOp, ::mlir::nvgpu::MmaSparseSyncOp, ::mlir::nvgpu::MmaSyncOp, ::mlir::nvgpu::RcpOp, ::mlir::nvgpu::TmaAsyncLoadOp, ::mlir::nvgpu::TmaAsyncStoreOp, ::mlir::nvgpu::TmaCreateDescriptorOp, ::mlir::nvgpu::TmaFenceOp, ::mlir::nvgpu::TmaPrefetchOp, ::mlir::nvgpu::WarpgroupGenerateDescriptorOp, ::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp, ::mlir::nvgpu::WarpgroupMmaOp, ::mlir::nvgpu::WarpgroupMmaStoreOp #endif // GET_OP_LIST #ifdef GET_OP_CLASSES #undef GET_OP_CLASSES //===----------------------------------------------------------------------===// // Local Utility Method Definitions //===----------------------------------------------------------------------===// static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps1( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps2( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps3( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps4( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::DeviceAsyncTokenType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be device async token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps5( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::DeviceAsyncTokenType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of device async token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps6( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps7( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::MBarrierGroupType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be mbarrier barrier type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps8( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger(1)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 1-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps9( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::MBarrierTokenType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be , but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps10( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 32-bit signless integer or 64-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps11( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 2))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of 16-bit signless integer values of length 2, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps12( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of 32-bit float values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps13( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::TensorMapDescriptorType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be TensorMap descriptor, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps14( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger(16)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 16-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps15( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::UnrankedMemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be unranked.memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps16( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be Warpgroup matrix descriptor type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps17( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::nvgpu::WarpgroupAccumulatorType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be , but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps1( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(attr).getType()))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: index attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps1( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps1(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps2( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::UnitAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: unit attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps2( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps2(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps3( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(32))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 32-bit signless integer attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps3( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps3(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps4( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::BoolAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: bool attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps4( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps4(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps5( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); })))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 64-bit integer array attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps5( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps5(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps6( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::NVVM::FPRoundingModeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: NVVM FPRoundingMode kind"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps6( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps6(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps7( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 64-bit signless integer attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps7( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_NVGPUOps7(attr, attrName, [op]() { return op->emitOpError(); }); } namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::DeviceAsyncCopyOp definitions //===----------------------------------------------------------------------===// namespace detail { DeviceAsyncCopyOpGenericAdaptorBase::DeviceAsyncCopyOpGenericAdaptorBase(DeviceAsyncCopyOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DeviceAsyncCopyOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::llvm::APInt DeviceAsyncCopyOpGenericAdaptorBase::getDstElements() { auto attr = getDstElementsAttr(); return attr.getValue(); } ::std::optional DeviceAsyncCopyOpGenericAdaptorBase::getBypassL1() { auto attr = getBypassL1Attr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail DeviceAsyncCopyOpAdaptor::DeviceAsyncCopyOpAdaptor(DeviceAsyncCopyOp op) : DeviceAsyncCopyOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DeviceAsyncCopyOpAdaptor::verify(::mlir::Location loc) { auto tblgen_bypassL1 = getProperties().bypassL1; (void)tblgen_bypassL1; auto tblgen_dstElements = getProperties().dstElements; (void)tblgen_dstElements; if (!tblgen_dstElements) return emitError(loc, "'nvgpu.device_async_copy' op requires attribute 'dstElements'"); if (tblgen_dstElements && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_dstElements))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_dstElements).getType()))))) return emitError(loc, "'nvgpu.device_async_copy' op attribute 'dstElements' failed to satisfy constraint: index attribute"); if (tblgen_bypassL1 && !((::llvm::isa<::mlir::UnitAttr>(tblgen_bypassL1)))) return emitError(loc, "'nvgpu.device_async_copy' op attribute 'bypassL1' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } std::pair DeviceAsyncCopyOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange DeviceAsyncCopyOp::getDstIndicesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange DeviceAsyncCopyOp::getSrcIndicesMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange DeviceAsyncCopyOp::getSrcElementsMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult DeviceAsyncCopyOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.bypassL1; auto attr = dict.get("bypassL1"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `bypassL1` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.dstElements; auto attr = dict.get("dstElements"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dstElements` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DeviceAsyncCopyOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.bypassL1; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("bypassL1", propStorage)); } { const auto &propStorage = prop.dstElements; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dstElements", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DeviceAsyncCopyOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.bypassL1.getAsOpaquePointer()), llvm::hash_value(prop.dstElements.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional DeviceAsyncCopyOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "bypassL1") return prop.bypassL1; if (name == "dstElements") return prop.dstElements; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void DeviceAsyncCopyOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "bypassL1") { prop.bypassL1 = ::llvm::dyn_cast_or_null>(value); return; } if (name == "dstElements") { prop.dstElements = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void DeviceAsyncCopyOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.bypassL1) attrs.append("bypassL1", prop.bypassL1); if (prop.dstElements) attrs.append("dstElements", prop.dstElements); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult DeviceAsyncCopyOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getBypassL1AttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "bypassL1", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getDstElementsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps1(attr, "dstElements", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncCopyOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.bypassL1))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.dstElements))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void DeviceAsyncCopyOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.bypassL1); writer.writeAttribute(prop.dstElements); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::llvm::APInt DeviceAsyncCopyOp::getDstElements() { auto attr = getDstElementsAttr(); return attr.getValue(); } ::std::optional DeviceAsyncCopyOp::getBypassL1() { auto attr = getBypassL1Attr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void DeviceAsyncCopyOp::setDstElements(::llvm::APInt attrValue) { getProperties().dstElements = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), attrValue); } void DeviceAsyncCopyOp::setBypassL1(bool attrValue) { auto &odsProp = getProperties().bypassL1; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = dstElements; if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } odsState.addTypes(asyncToken); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = dstElements; if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = dstElements; if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements); if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } odsState.addTypes(asyncToken); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements); if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { odsState.addOperands(dst); odsState.addOperands(dstIndices); odsState.addOperands(src); odsState.addOperands(srcIndices); if (srcElements) odsState.addOperands(srcElements); ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndices.size()), 1, static_cast(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements); if (bypassL1) { odsState.getOrAddProperties().bypassL1 = bypassL1; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dst), std::forward(dstIndices), std::forward(src), std::forward(srcIndices), std::forward(dstElements), std::forward(srcElements), std::forward(bypassL1)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DeviceAsyncCopyOp::verifyInvariantsImpl() { auto tblgen_bypassL1 = getProperties().bypassL1; (void)tblgen_bypassL1; auto tblgen_dstElements = getProperties().dstElements; (void)tblgen_dstElements; if (!tblgen_dstElements) return emitOpError("requires attribute 'dstElements'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps1(*this, tblgen_dstElements, "dstElements"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_bypassL1, "bypassL1"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncCopyOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult DeviceAsyncCopyOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::DeviceAsyncTokenType>(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult DeviceAsyncCopyOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc; (void)srcOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> srcIndicesOperands; ::llvm::SMLoc srcIndicesOperandsLoc; (void)srcIndicesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc; (void)dstOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> dstIndicesOperands; ::llvm::SMLoc dstIndicesOperandsLoc; (void)dstIndicesOperandsLoc; ::mlir::IntegerAttr dstElementsAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> srcElementsOperands; ::llvm::SMLoc srcElementsOperandsLoc; (void)srcElementsOperandsLoc; ::mlir::Type srcRawType{}; ::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1); ::mlir::Type dstRawType{}; ::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1); srcOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); srcIndicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(srcIndicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dstOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); dstIndicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(dstIndicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(dstElementsAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (dstElementsAttr) result.getOrAddProperties().dstElements = dstElementsAttr; if (::mlir::succeeded(parser.parseOptionalComma())) { { srcElementsOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); srcElementsOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcRawType = type; } if (parser.parseKeyword("to")) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, static_cast(dstIndicesOperands.size()), 1, static_cast(srcIndicesOperands.size()), static_cast(srcElementsOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>(); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dstIndicesOperands, odsBuildableType1, dstIndicesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(srcIndicesOperands, odsBuildableType1, srcIndicesOperandsLoc, result.operands)) return ::mlir::failure(); if (!srcElementsOperands.empty()) { if (parser.resolveOperands(srcElementsOperands, odsBuildableType1, srcElementsOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void DeviceAsyncCopyOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrc(); _odsPrinter << "["; _odsPrinter << getSrcIndices(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDst(); _odsPrinter << "["; _odsPrinter << getDstIndices(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getDstElementsAttr()); if (getSrcElements()) { _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getSrcElements()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("dstElements"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSrc().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "to"; _odsPrinter << ' '; { auto type = getDst().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void DeviceAsyncCopyOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } { auto valueRange = getODSOperandIndexAndLength(2); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncCopyOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::DeviceAsyncCopyOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::DeviceAsyncCreateGroupOp definitions //===----------------------------------------------------------------------===// namespace detail { DeviceAsyncCreateGroupOpGenericAdaptorBase::DeviceAsyncCreateGroupOpGenericAdaptorBase(DeviceAsyncCreateGroupOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DeviceAsyncCreateGroupOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail DeviceAsyncCreateGroupOpAdaptor::DeviceAsyncCreateGroupOpAdaptor(DeviceAsyncCreateGroupOp op) : DeviceAsyncCreateGroupOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DeviceAsyncCreateGroupOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair DeviceAsyncCreateGroupOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange DeviceAsyncCreateGroupOp::getInputTokensMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult DeviceAsyncCreateGroupOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DeviceAsyncCreateGroupOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DeviceAsyncCreateGroupOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional DeviceAsyncCreateGroupOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void DeviceAsyncCreateGroupOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void DeviceAsyncCreateGroupOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) { odsState.addOperands(inputTokens); odsState.addTypes(asyncToken); } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(inputTokens)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(inputTokens)); } void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCreateGroupOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(DeviceAsyncCreateGroupOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult DeviceAsyncCreateGroupOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::DeviceAsyncTokenType>(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult DeviceAsyncCreateGroupOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> inputTokensOperands; ::llvm::SMLoc inputTokensOperandsLoc; (void)inputTokensOperandsLoc; inputTokensOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(inputTokensOperands)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>(); result.addTypes(odsBuildableType0); if (parser.resolveOperands(inputTokensOperands, odsBuildableType0, inputTokensOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DeviceAsyncCreateGroupOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getInputTokens(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncCreateGroupOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::DeviceAsyncWaitOp definitions //===----------------------------------------------------------------------===// namespace detail { DeviceAsyncWaitOpGenericAdaptorBase::DeviceAsyncWaitOpGenericAdaptorBase(DeviceAsyncWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional DeviceAsyncWaitOpGenericAdaptorBase::getNumGroups() { auto attr = getNumGroupsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } } // namespace detail DeviceAsyncWaitOpAdaptor::DeviceAsyncWaitOpAdaptor(DeviceAsyncWaitOp op) : DeviceAsyncWaitOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DeviceAsyncWaitOpAdaptor::verify(::mlir::Location loc) { auto tblgen_numGroups = getProperties().numGroups; (void)tblgen_numGroups; if (tblgen_numGroups && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_numGroups))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_numGroups).getType().isSignlessInteger(32))))) return emitError(loc, "'nvgpu.device_async_wait' op attribute 'numGroups' failed to satisfy constraint: 32-bit signless integer attribute"); return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.numGroups; auto attr = dict.get("numGroups"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `numGroups` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute DeviceAsyncWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.numGroups; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("numGroups", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DeviceAsyncWaitOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.numGroups.getAsOpaquePointer())); } std::optional DeviceAsyncWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "numGroups") return prop.numGroups; return std::nullopt; } void DeviceAsyncWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "numGroups") { prop.numGroups = ::llvm::dyn_cast_or_null>(value); return; } } void DeviceAsyncWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.numGroups) attrs.append("numGroups", prop.numGroups); } ::llvm::LogicalResult DeviceAsyncWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getNumGroupsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "numGroups", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncWaitOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.numGroups))) return ::mlir::failure(); return ::mlir::success(); } void DeviceAsyncWaitOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.numGroups); } ::std::optional DeviceAsyncWaitOp::getNumGroups() { auto attr = getNumGroupsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } void DeviceAsyncWaitOp::setNumGroups(::std::optional attrValue) { auto &odsProp = getProperties().numGroups; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), *attrValue); else odsProp = nullptr; } void DeviceAsyncWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { odsState.addOperands(asyncDependencies); if (numGroups) { odsState.getOrAddProperties().numGroups = numGroups; } } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncDependencies), std::forward(numGroups)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { return create(builder, builder.getLoc(), std::forward(asyncDependencies), std::forward(numGroups)); } void DeviceAsyncWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { odsState.addOperands(asyncDependencies); if (numGroups) { odsState.getOrAddProperties().numGroups = numGroups; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(numGroups)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(numGroups)); } void DeviceAsyncWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DeviceAsyncWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DeviceAsyncWaitOp::verifyInvariantsImpl() { auto tblgen_numGroups = getProperties().numGroups; (void)tblgen_numGroups; if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_numGroups, "numGroups"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DeviceAsyncWaitOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult DeviceAsyncWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand asyncDependenciesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> asyncDependenciesOperands(&asyncDependenciesRawOperand, 1); ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(asyncDependenciesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>(); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DeviceAsyncWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getAsyncDependencies(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncWaitOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::DeviceAsyncWaitOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::LdMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { LdMatrixOpGenericAdaptorBase::LdMatrixOpGenericAdaptorBase(LdMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LdMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } bool LdMatrixOpGenericAdaptorBase::getTranspose() { auto attr = getTransposeAttr(); return attr.getValue(); } uint32_t LdMatrixOpGenericAdaptorBase::getNumTiles() { auto attr = getNumTilesAttr(); return attr.getValue().getZExtValue(); } } // namespace detail LdMatrixOpAdaptor::LdMatrixOpAdaptor(LdMatrixOp op) : LdMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LdMatrixOpAdaptor::verify(::mlir::Location loc) { auto tblgen_numTiles = getProperties().numTiles; (void)tblgen_numTiles; if (!tblgen_numTiles) return emitError(loc, "'nvgpu.ldmatrix' op requires attribute 'numTiles'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (!tblgen_transpose) return emitError(loc, "'nvgpu.ldmatrix' op requires attribute 'transpose'"); if (tblgen_transpose && !((::llvm::isa<::mlir::BoolAttr>(tblgen_transpose)))) return emitError(loc, "'nvgpu.ldmatrix' op attribute 'transpose' failed to satisfy constraint: bool attribute"); if (tblgen_numTiles && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_numTiles))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_numTiles).getType().isSignlessInteger(32))))) return emitError(loc, "'nvgpu.ldmatrix' op attribute 'numTiles' failed to satisfy constraint: 32-bit signless integer attribute"); return ::mlir::success(); } std::pair LdMatrixOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange LdMatrixOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult LdMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.numTiles; auto attr = dict.get("numTiles"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `numTiles` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transpose; auto attr = dict.get("transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transpose` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LdMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.numTiles; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("numTiles", propStorage)); } { const auto &propStorage = prop.transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transpose", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LdMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.numTiles.getAsOpaquePointer()), llvm::hash_value(prop.transpose.getAsOpaquePointer())); } std::optional LdMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "numTiles") return prop.numTiles; if (name == "transpose") return prop.transpose; return std::nullopt; } void LdMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "numTiles") { prop.numTiles = ::llvm::dyn_cast_or_null>(value); return; } if (name == "transpose") { prop.transpose = ::llvm::dyn_cast_or_null>(value); return; } } void LdMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.numTiles) attrs.append("numTiles", prop.numTiles); if (prop.transpose) attrs.append("transpose", prop.transpose); } ::llvm::LogicalResult LdMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getNumTilesAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "numTiles", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "transpose", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LdMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.numTiles))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.transpose))) return ::mlir::failure(); return ::mlir::success(); } void LdMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.numTiles); writer.writeAttribute(prop.transpose); } bool LdMatrixOp::getTranspose() { auto attr = getTransposeAttr(); return attr.getValue(); } uint32_t LdMatrixOp::getNumTiles() { auto attr = getNumTilesAttr(); return attr.getValue().getZExtValue(); } void LdMatrixOp::setTranspose(bool attrValue) { getProperties().transpose = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue); } void LdMatrixOp::setNumTiles(uint32_t attrValue) { getProperties().numTiles = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue); } void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().transpose = transpose; odsState.getOrAddProperties().numTiles = numTiles; odsState.addTypes(res); } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { return create(builder, builder.getLoc(), std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); } void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().transpose = transpose; odsState.getOrAddProperties().numTiles = numTiles; assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); } void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().transpose = odsBuilder.getBoolAttr(transpose); odsState.getOrAddProperties().numTiles = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), numTiles); odsState.addTypes(res); } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { return create(builder, builder.getLoc(), std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); } void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().transpose = odsBuilder.getBoolAttr(transpose); odsState.getOrAddProperties().numTiles = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), numTiles); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(transpose), std::forward(numTiles)); } void LdMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LdMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult LdMatrixOp::verifyInvariantsImpl() { auto tblgen_numTiles = getProperties().numTiles; (void)tblgen_numTiles; if (!tblgen_numTiles) return emitOpError("requires attribute 'numTiles'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (!tblgen_transpose) return emitOpError("requires attribute 'transpose'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_transpose, "transpose"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_numTiles, "numTiles"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((((*this->getOperation()).getNumResults() > 0)) && (((*this->getOperation()).getNumOperands() > 0)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getResult(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType())))) && ((getElementTypeOrSelf((*this->getOperation()).getResult(0)) == getElementTypeOrSelf((*this->getOperation()).getOperand(0)))))) return emitOpError("failed to verify that srcMemref and res have same element type"); return ::mlir::success(); } ::llvm::LogicalResult LdMatrixOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult LdMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcMemrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcMemrefOperands(&srcMemrefRawOperand, 1); ::llvm::SMLoc srcMemrefOperandsLoc; (void)srcMemrefOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::Type srcMemrefRawType{}; ::llvm::ArrayRef<::mlir::Type> srcMemrefTypes(&srcMemrefRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); srcMemrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcMemrefRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcMemrefRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resTypes); if (parser.resolveOperands(srcMemrefOperands, srcMemrefTypes, srcMemrefOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void LdMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrcMemref(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSrcMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void LdMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::LdMatrixOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::LdMatrixOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierArriveExpectTxOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierArriveExpectTxOpGenericAdaptorBase::MBarrierArriveExpectTxOpGenericAdaptorBase(MBarrierArriveExpectTxOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair MBarrierArriveExpectTxOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail MBarrierArriveExpectTxOpAdaptor::MBarrierArriveExpectTxOpAdaptor(MBarrierArriveExpectTxOp op) : MBarrierArriveExpectTxOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierArriveExpectTxOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair MBarrierArriveExpectTxOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange MBarrierArriveExpectTxOp::getPredicateMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult MBarrierArriveExpectTxOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierArriveExpectTxOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierArriveExpectTxOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierArriveExpectTxOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierArriveExpectTxOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierArriveExpectTxOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { odsState.addOperands(barriers); odsState.addOperands(txcount); odsState.addOperands(mbarId); if (predicate) odsState.addOperands(predicate); } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(txcount), std::forward(mbarId), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(txcount), std::forward(mbarId), std::forward(predicate)); } void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { odsState.addOperands(barriers); odsState.addOperands(txcount); odsState.addOperands(mbarId); if (predicate) odsState.addOperands(predicate); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(txcount), std::forward(mbarId), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(txcount), std::forward(mbarId), std::forward(predicate)); } void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MBarrierArriveExpectTxOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand txcountRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> txcountOperands(&txcountRawOperand, 1); ::llvm::SMLoc txcountOperandsLoc; (void)txcountOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands; ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); txcountOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(txcountRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("predicate")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); { predicateOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); predicateOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(txcountOperands, odsBuildableType0, txcountOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); if (!predicateOperands.empty()) { if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void MBarrierArriveExpectTxOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTxcount(); if (getPredicate()) { _odsPrinter << ","; _odsPrinter << ' ' << "predicate"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getPredicate()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveExpectTxOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierArriveNoCompleteOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierArriveNoCompleteOpGenericAdaptorBase::MBarrierArriveNoCompleteOpGenericAdaptorBase(MBarrierArriveNoCompleteOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierArriveNoCompleteOpAdaptor::MBarrierArriveNoCompleteOpAdaptor(MBarrierArriveNoCompleteOp op) : MBarrierArriveNoCompleteOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierArriveNoCompleteOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierArriveNoCompleteOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierArriveNoCompleteOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierArriveNoCompleteOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierArriveNoCompleteOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierArriveNoCompleteOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierArriveNoCompleteOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { odsState.addOperands(barriers); odsState.addOperands(mbarId); odsState.addOperands(count); odsState.addTypes(token); } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(token), std::forward(barriers), std::forward(mbarId), std::forward(count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { return create(builder, builder.getLoc(), std::forward(token), std::forward(barriers), std::forward(mbarId), std::forward(count)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { odsState.addOperands(barriers); odsState.addOperands(mbarId); odsState.addOperands(count); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(mbarId), std::forward(count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(mbarId), std::forward(count)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { odsState.addOperands(barriers); odsState.addOperands(mbarId); odsState.addOperands(count); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(mbarId), std::forward(count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(mbarId), std::forward(count)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult MBarrierArriveNoCompleteOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::MBarrierTokenType>(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult MBarrierArriveNoCompleteOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand countRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> countOperands(&countRawOperand, 1); ::llvm::SMLoc countOperandsLoc; (void)countOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); ::mlir::Type tokenRawType{}; ::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); countOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(countRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierTokenType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tokenRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(tokenTypes); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(countOperands, odsBuildableType0, countOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MBarrierArriveNoCompleteOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCount(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getToken().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveNoCompleteOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierArriveOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierArriveOpGenericAdaptorBase::MBarrierArriveOpGenericAdaptorBase(MBarrierArriveOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierArriveOpAdaptor::MBarrierArriveOpAdaptor(MBarrierArriveOp op) : MBarrierArriveOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierArriveOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierArriveOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierArriveOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierArriveOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierArriveOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierArriveOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierArriveOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierArriveOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(mbarId); odsState.addTypes(token); } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(token), std::forward(barriers), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(token), std::forward(barriers), std::forward(mbarId)); } void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(mbarId); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(mbarId)); } void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(mbarId); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(mbarId)); } void MBarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void MBarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierArriveOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierArriveOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult MBarrierArriveOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::MBarrierTokenType>(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult MBarrierArriveOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); ::mlir::Type tokenRawType{}; ::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierTokenType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tokenRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(tokenTypes); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MBarrierArriveOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getToken().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierCreateOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierCreateOpGenericAdaptorBase::MBarrierCreateOpGenericAdaptorBase(MBarrierCreateOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierCreateOpAdaptor::MBarrierCreateOpAdaptor(MBarrierCreateOp op) : MBarrierCreateOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierCreateOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierCreateOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierCreateOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierCreateOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierCreateOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierCreateOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierCreateOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierCreateOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierCreateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type barriers) { odsState.addTypes(barriers); } MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type barriers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type barriers) { return create(builder, builder.getLoc(), std::forward(barriers)); } void MBarrierCreateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void MBarrierCreateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierCreateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierCreateOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierCreateOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MBarrierCreateOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } result.addTypes(barriersTypes); return ::mlir::success(); } void MBarrierCreateOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierCreateOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierGetOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierGetOpGenericAdaptorBase::MBarrierGetOpGenericAdaptorBase(MBarrierGetOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierGetOpAdaptor::MBarrierGetOpAdaptor(MBarrierGetOp op) : MBarrierGetOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierGetOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierGetOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierGetOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierGetOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierGetOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierGetOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierGetOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierGetOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierGetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(mbarId); odsState.addTypes(mbarrierPointer); } MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(mbarrierPointer), std::forward(barriers), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(mbarrierPointer), std::forward(barriers), std::forward(mbarId)); } void MBarrierGetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(mbarId); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(mbarId)); } void MBarrierGetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierGetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierGetOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierGetOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MBarrierGetOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); ::mlir::Type mbarrierPointerRawType{}; ::llvm::ArrayRef<::mlir::Type> mbarrierPointerTypes(&mbarrierPointerRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); mbarrierPointerRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(mbarrierPointerTypes); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MBarrierGetOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getMbarrierPointer().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierGetOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierInitOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierInitOpGenericAdaptorBase::MBarrierInitOpGenericAdaptorBase(MBarrierInitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair MBarrierInitOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail MBarrierInitOpAdaptor::MBarrierInitOpAdaptor(MBarrierInitOp op) : MBarrierInitOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierInitOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair MBarrierInitOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange MBarrierInitOp::getPredicateMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult MBarrierInitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierInitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierInitOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierInitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierInitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierInitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierInitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierInitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { odsState.addOperands(barriers); odsState.addOperands(count); odsState.addOperands(mbarId); if (predicate) odsState.addOperands(predicate); } MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(count), std::forward(mbarId), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(count), std::forward(mbarId), std::forward(predicate)); } void MBarrierInitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { odsState.addOperands(barriers); odsState.addOperands(count); odsState.addOperands(mbarId); if (predicate) odsState.addOperands(predicate); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(count), std::forward(mbarId), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(count), std::forward(mbarId), std::forward(predicate)); } void MBarrierInitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierInitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierInitOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierInitOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MBarrierInitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand countRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> countOperands(&countRawOperand, 1); ::llvm::SMLoc countOperandsLoc; (void)countOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands; ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); countOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(countRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("predicate")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); { predicateOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); predicateOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(countOperands, odsBuildableType0, countOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); if (!predicateOperands.empty()) { if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void MBarrierInitOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCount(); if (getPredicate()) { _odsPrinter << ","; _odsPrinter << ' ' << "predicate"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getPredicate()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierInitOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierTestWaitOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierTestWaitOpGenericAdaptorBase::MBarrierTestWaitOpGenericAdaptorBase(MBarrierTestWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierTestWaitOpAdaptor::MBarrierTestWaitOpAdaptor(MBarrierTestWaitOp op) : MBarrierTestWaitOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierTestWaitOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierTestWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierTestWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierTestWaitOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierTestWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierTestWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierTestWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierTestWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(token); odsState.addOperands(mbarId); odsState.addTypes(waitComplete); } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(waitComplete), std::forward(barriers), std::forward(token), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(waitComplete), std::forward(barriers), std::forward(token), std::forward(mbarId)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(token); odsState.addOperands(mbarId); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(token), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(token), std::forward(mbarId)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(token); odsState.addOperands(mbarId); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(token), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(token), std::forward(mbarId)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierTestWaitOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierTestWaitOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult MBarrierTestWaitOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIntegerType(1); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult MBarrierTestWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tokenRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tokenOperands(&tokenRawOperand, 1); ::llvm::SMLoc tokenOperandsLoc; (void)tokenOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); ::mlir::Type tokenRawType{}; ::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tokenOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tokenRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierTokenType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tokenRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tokenOperands, tokenTypes, tokenOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType1, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MBarrierTestWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getToken(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getToken().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierTestWaitOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MBarrierTryWaitParityOp definitions //===----------------------------------------------------------------------===// namespace detail { MBarrierTryWaitParityOpGenericAdaptorBase::MBarrierTryWaitParityOpGenericAdaptorBase(MBarrierTryWaitParityOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail MBarrierTryWaitParityOpAdaptor::MBarrierTryWaitParityOpAdaptor(MBarrierTryWaitParityOp op) : MBarrierTryWaitParityOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MBarrierTryWaitParityOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult MBarrierTryWaitParityOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MBarrierTryWaitParityOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MBarrierTryWaitParityOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MBarrierTryWaitParityOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MBarrierTryWaitParityOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MBarrierTryWaitParityOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(phaseParity); odsState.addOperands(ticks); odsState.addOperands(mbarId); } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(barriers), std::forward(phaseParity), std::forward(ticks), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(barriers), std::forward(phaseParity), std::forward(ticks), std::forward(mbarId)); } void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { odsState.addOperands(barriers); odsState.addOperands(phaseParity); odsState.addOperands(ticks); odsState.addOperands(mbarId); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(barriers), std::forward(phaseParity), std::forward(ticks), std::forward(mbarId)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(barriers), std::forward(phaseParity), std::forward(ticks), std::forward(mbarId)); } void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MBarrierTryWaitParityOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand phaseParityRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> phaseParityOperands(&phaseParityRawOperand, 1); ::llvm::SMLoc phaseParityOperandsLoc; (void)phaseParityOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand ticksRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> ticksOperands(&ticksRawOperand, 1); ::llvm::SMLoc ticksOperandsLoc; (void)ticksOperandsLoc; ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); phaseParityOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(phaseParityRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); ticksOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(ticksRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(phaseParityOperands, odsBuildableType0, phaseParityOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(ticksOperands, odsBuildableType1, ticksOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType1, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MBarrierTryWaitParityOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getPhaseParity(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTicks(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierTryWaitParityOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MmaSparseSyncOp definitions //===----------------------------------------------------------------------===// namespace detail { MmaSparseSyncOpGenericAdaptorBase::MmaSparseSyncOpGenericAdaptorBase(MmaSparseSyncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::ArrayAttr MmaSparseSyncOpGenericAdaptorBase::getMmaShape() { auto attr = getMmaShapeAttr(); return attr; } ::mlir::IntegerAttr MmaSparseSyncOpGenericAdaptorBase::getSparsitySelectorAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::IntegerAttr>(getProperties().sparsitySelector); return attr; } uint32_t MmaSparseSyncOpGenericAdaptorBase::getSparsitySelector() { auto attr = getSparsitySelectorAttr(); return attr.getValue().getZExtValue(); } ::std::optional MmaSparseSyncOpGenericAdaptorBase::getTf32Enabled() { auto attr = getTf32EnabledAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail MmaSparseSyncOpAdaptor::MmaSparseSyncOpAdaptor(MmaSparseSyncOp op) : MmaSparseSyncOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MmaSparseSyncOpAdaptor::verify(::mlir::Location loc) { auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape; if (!tblgen_mmaShape) return emitError(loc, "'nvgpu.mma.sp.sync' op requires attribute 'mmaShape'"); auto tblgen_sparsitySelector = getProperties().sparsitySelector; (void)tblgen_sparsitySelector; auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled; if (tblgen_mmaShape && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_mmaShape))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_mmaShape), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); })))) return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'mmaShape' failed to satisfy constraint: 64-bit integer array attribute"); if (tblgen_sparsitySelector && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_sparsitySelector))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_sparsitySelector).getType().isSignlessInteger(32))))) return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'sparsitySelector' failed to satisfy constraint: 32-bit signless integer attribute"); if (tblgen_tf32Enabled && !((::llvm::isa<::mlir::UnitAttr>(tblgen_tf32Enabled)))) return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'tf32Enabled' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult MmaSparseSyncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.mmaShape; auto attr = dict.get("mmaShape"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `mmaShape` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.sparsitySelector; auto attr = dict.get("sparsitySelector"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `sparsitySelector` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.tf32Enabled; auto attr = dict.get("tf32Enabled"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `tf32Enabled` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute MmaSparseSyncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.mmaShape; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("mmaShape", propStorage)); } { const auto &propStorage = prop.sparsitySelector; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("sparsitySelector", propStorage)); } { const auto &propStorage = prop.tf32Enabled; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("tf32Enabled", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MmaSparseSyncOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.mmaShape.getAsOpaquePointer()), llvm::hash_value(prop.sparsitySelector.getAsOpaquePointer()), llvm::hash_value(prop.tf32Enabled.getAsOpaquePointer())); } std::optional MmaSparseSyncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "mmaShape") return prop.mmaShape; if (name == "sparsitySelector") return prop.sparsitySelector; if (name == "tf32Enabled") return prop.tf32Enabled; return std::nullopt; } void MmaSparseSyncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "mmaShape") { prop.mmaShape = ::llvm::dyn_cast_or_null>(value); return; } if (name == "sparsitySelector") { prop.sparsitySelector = ::llvm::dyn_cast_or_null>(value); return; } if (name == "tf32Enabled") { prop.tf32Enabled = ::llvm::dyn_cast_or_null>(value); return; } } void MmaSparseSyncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.mmaShape) attrs.append("mmaShape", prop.mmaShape); if (prop.sparsitySelector) attrs.append("sparsitySelector", prop.sparsitySelector); if (prop.tf32Enabled) attrs.append("tf32Enabled", prop.tf32Enabled); } ::llvm::LogicalResult MmaSparseSyncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getMmaShapeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(attr, "mmaShape", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getSparsitySelectorAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "sparsitySelector", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTf32EnabledAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "tf32Enabled", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult MmaSparseSyncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.mmaShape))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.sparsitySelector))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.tf32Enabled))) return ::mlir::failure(); return ::mlir::success(); } void MmaSparseSyncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.mmaShape); writer.writeOptionalAttribute(prop.sparsitySelector); writer.writeOptionalAttribute(prop.tf32Enabled); } ::mlir::ArrayAttr MmaSparseSyncOp::getMmaShape() { auto attr = getMmaShapeAttr(); return attr; } uint32_t MmaSparseSyncOp::getSparsitySelector() { auto attr = getSparsitySelectorAttr(); return attr.getValue().getZExtValue(); } ::std::optional MmaSparseSyncOp::getTf32Enabled() { auto attr = getTf32EnabledAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void MmaSparseSyncOp::setSparsitySelector(uint32_t attrValue) { getProperties().sparsitySelector = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue); } void MmaSparseSyncOp::setTf32Enabled(bool attrValue) { auto &odsProp = getProperties().tf32Enabled; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, Value sparseMetadata, ArrayRef mmaShape) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, Value sparseMetadata, ArrayRef mmaShape) { return create(builder, builder.getLoc(), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.addOperands(sparseMetadata); odsState.getOrAddProperties().mmaShape = mmaShape; if (sparsitySelector) { odsState.getOrAddProperties().sparsitySelector = sparsitySelector; } if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } odsState.addTypes(res); } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.addOperands(sparseMetadata); odsState.getOrAddProperties().mmaShape = mmaShape; if (sparsitySelector) { odsState.getOrAddProperties().sparsitySelector = sparsitySelector; } if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.addOperands(sparseMetadata); odsState.getOrAddProperties().mmaShape = mmaShape; odsState.getOrAddProperties().sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), sparsitySelector); if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } odsState.addTypes(res); } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.addOperands(sparseMetadata); odsState.getOrAddProperties().mmaShape = mmaShape; odsState.getOrAddProperties().sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), sparsitySelector); if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(sparseMetadata), std::forward(mmaShape), std::forward(sparsitySelector), std::forward(tf32Enabled)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MmaSparseSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void MmaSparseSyncOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.sparsitySelector) properties.sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 0); } ::llvm::LogicalResult MmaSparseSyncOp::verifyInvariantsImpl() { auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape; if (!tblgen_mmaShape) return emitOpError("requires attribute 'mmaShape'"); auto tblgen_sparsitySelector = getProperties().sparsitySelector; (void)tblgen_sparsitySelector; auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled; if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(*this, tblgen_mmaShape, "mmaShape"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_sparsitySelector, "sparsitySelector"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_tf32Enabled, "tf32Enabled"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getOperation()).getNumOperands() > 1)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(1).getType()))) && ((::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(0)) == ::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(1)))))) return emitOpError("failed to verify that matrixA and matrixB have same element type"); return ::mlir::success(); } ::llvm::LogicalResult MmaSparseSyncOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult MmaSparseSyncOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand matrixARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixAOperands(&matrixARawOperand, 1); ::llvm::SMLoc matrixAOperandsLoc; (void)matrixAOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixBOperands(&matrixBRawOperand, 1); ::llvm::SMLoc matrixBOperandsLoc; (void)matrixBOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc; (void)matrixCOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand sparseMetadataRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sparseMetadataOperands(&sparseMetadataRawOperand, 1); ::llvm::SMLoc sparseMetadataOperandsLoc; (void)sparseMetadataOperandsLoc; ::mlir::Type matrixARawType{}; ::llvm::ArrayRef<::mlir::Type> matrixATypes(&matrixARawType, 1); ::mlir::Type matrixBRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixBTypes(&matrixBRawType, 1); ::mlir::Type matrixCRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); if (parser.parseLParen()) return ::mlir::failure(); matrixAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixARawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixBRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixCRawOperand)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); if (parser.parseKeyword("metadata")) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); sparseMetadataOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sparseMetadataRawOperand)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixARawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixBRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixCRawType = type; } if (parser.parseRParen()) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } ::mlir::Type odsBuildableType0 = ::mlir::VectorType::get({2},parser.getBuilder().getI16Type()); result.addTypes(resTypes); if (parser.resolveOperands(matrixAOperands, matrixATypes, matrixAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixBOperands, matrixBTypes, matrixBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(sparseMetadataOperands, odsBuildableType0, sparseMetadataOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MmaSparseSyncOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << "("; _odsPrinter << getMatrixA(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrixB(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrixC(); _odsPrinter << ")"; _odsPrinter << ' ' << "metadata"; _odsPrinter << "("; _odsPrinter << getSparseMetadata(); _odsPrinter << ")"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getSparsitySelectorAttr(); if(attr && (attr == odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 0))) elidedAttrs.push_back("sparsitySelector"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' ' << "("; { auto type = getMatrixA().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrixB().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrixC().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ")"; _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void MmaSparseSyncOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MmaSparseSyncOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::MmaSparseSyncOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::MmaSyncOp definitions //===----------------------------------------------------------------------===// namespace detail { MmaSyncOpGenericAdaptorBase::MmaSyncOpGenericAdaptorBase(MmaSyncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::ArrayAttr MmaSyncOpGenericAdaptorBase::getMmaShape() { auto attr = getMmaShapeAttr(); return attr; } ::std::optional MmaSyncOpGenericAdaptorBase::getTf32Enabled() { auto attr = getTf32EnabledAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail MmaSyncOpAdaptor::MmaSyncOpAdaptor(MmaSyncOp op) : MmaSyncOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MmaSyncOpAdaptor::verify(::mlir::Location loc) { auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape; if (!tblgen_mmaShape) return emitError(loc, "'nvgpu.mma.sync' op requires attribute 'mmaShape'"); auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled; if (tblgen_mmaShape && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_mmaShape))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_mmaShape), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); })))) return emitError(loc, "'nvgpu.mma.sync' op attribute 'mmaShape' failed to satisfy constraint: 64-bit integer array attribute"); if (tblgen_tf32Enabled && !((::llvm::isa<::mlir::UnitAttr>(tblgen_tf32Enabled)))) return emitError(loc, "'nvgpu.mma.sync' op attribute 'tf32Enabled' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult MmaSyncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.mmaShape; auto attr = dict.get("mmaShape"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `mmaShape` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.tf32Enabled; auto attr = dict.get("tf32Enabled"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `tf32Enabled` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute MmaSyncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.mmaShape; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("mmaShape", propStorage)); } { const auto &propStorage = prop.tf32Enabled; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("tf32Enabled", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MmaSyncOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.mmaShape.getAsOpaquePointer()), llvm::hash_value(prop.tf32Enabled.getAsOpaquePointer())); } std::optional MmaSyncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "mmaShape") return prop.mmaShape; if (name == "tf32Enabled") return prop.tf32Enabled; return std::nullopt; } void MmaSyncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "mmaShape") { prop.mmaShape = ::llvm::dyn_cast_or_null>(value); return; } if (name == "tf32Enabled") { prop.tf32Enabled = ::llvm::dyn_cast_or_null>(value); return; } } void MmaSyncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.mmaShape) attrs.append("mmaShape", prop.mmaShape); if (prop.tf32Enabled) attrs.append("tf32Enabled", prop.tf32Enabled); } ::llvm::LogicalResult MmaSyncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getMmaShapeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(attr, "mmaShape", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTf32EnabledAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "tf32Enabled", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult MmaSyncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.mmaShape))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.tf32Enabled))) return ::mlir::failure(); return ::mlir::success(); } void MmaSyncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.mmaShape); writer.writeOptionalAttribute(prop.tf32Enabled); } ::mlir::ArrayAttr MmaSyncOp::getMmaShape() { auto attr = getMmaShapeAttr(); return attr; } ::std::optional MmaSyncOp::getTf32Enabled() { auto attr = getTf32EnabledAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void MmaSyncOp::setTf32Enabled(bool attrValue) { auto &odsProp = getProperties().tf32Enabled; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, ArrayAttr mmaShape) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, ArrayAttr mmaShape) { return create(builder, builder.getLoc(), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape)); } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, ArrayRef mmaShape, bool tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, ArrayRef mmaShape, bool tf32Enabled) { return create(builder, builder.getLoc(), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); } void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.getOrAddProperties().mmaShape = mmaShape; if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } odsState.addTypes(res); } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(res), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); } void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { odsState.addOperands(matrixA); odsState.addOperands(matrixB); odsState.addOperands(matrixC); odsState.getOrAddProperties().mmaShape = mmaShape; if (tf32Enabled) { odsState.getOrAddProperties().tf32Enabled = tf32Enabled; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(matrixA), std::forward(matrixB), std::forward(matrixC), std::forward(mmaShape), std::forward(tf32Enabled)); } void MmaSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MmaSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MmaSyncOp::verifyInvariantsImpl() { auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape; if (!tblgen_mmaShape) return emitOpError("requires attribute 'mmaShape'"); auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled; if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(*this, tblgen_mmaShape, "mmaShape"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_tf32Enabled, "tf32Enabled"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getOperation()).getNumOperands() > 1)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(1).getType()))) && ((::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(0)) == ::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(1)))))) return emitOpError("failed to verify that matrixA and matrixB have same element type"); return ::mlir::success(); } ::llvm::LogicalResult MmaSyncOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult MmaSyncOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand matrixARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixAOperands(&matrixARawOperand, 1); ::llvm::SMLoc matrixAOperandsLoc; (void)matrixAOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixBOperands(&matrixBRawOperand, 1); ::llvm::SMLoc matrixBOperandsLoc; (void)matrixBOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc; (void)matrixCOperandsLoc; ::mlir::Type matrixARawType{}; ::llvm::ArrayRef<::mlir::Type> matrixATypes(&matrixARawType, 1); ::mlir::Type matrixBRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixBTypes(&matrixBRawType, 1); ::mlir::Type matrixCRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); if (parser.parseLParen()) return ::mlir::failure(); matrixAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixARawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixBRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixCRawOperand)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixARawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixBRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixCRawType = type; } if (parser.parseRParen()) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } result.addTypes(resTypes); if (parser.resolveOperands(matrixAOperands, matrixATypes, matrixAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixBOperands, matrixBTypes, matrixBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MmaSyncOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << "("; _odsPrinter << getMatrixA(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrixB(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrixC(); _odsPrinter << ")"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' ' << "("; { auto type = getMatrixA().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrixB().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrixC().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ")"; _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void MmaSyncOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MmaSyncOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::MmaSyncOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::RcpOp definitions //===----------------------------------------------------------------------===// namespace detail { RcpOpGenericAdaptorBase::RcpOpGenericAdaptorBase(RcpOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::NVVM::FPRoundingModeAttr RcpOpGenericAdaptorBase::getRoundingAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::NVVM::FPRoundingModeAttr>(getProperties().rounding); return attr; } ::mlir::NVVM::FPRoundingMode RcpOpGenericAdaptorBase::getRounding() { auto attr = getRoundingAttr(); return attr.getValue(); } ::mlir::BoolAttr RcpOpGenericAdaptorBase::getApproxAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::BoolAttr>(getProperties().approx); return attr; } bool RcpOpGenericAdaptorBase::getApprox() { auto attr = getApproxAttr(); return attr.getValue(); } ::mlir::BoolAttr RcpOpGenericAdaptorBase::getFtzAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::BoolAttr>(getProperties().ftz); return attr; } bool RcpOpGenericAdaptorBase::getFtz() { auto attr = getFtzAttr(); return attr.getValue(); } } // namespace detail RcpOpAdaptor::RcpOpAdaptor(RcpOp op) : RcpOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult RcpOpAdaptor::verify(::mlir::Location loc) { auto tblgen_approx = getProperties().approx; (void)tblgen_approx; auto tblgen_ftz = getProperties().ftz; (void)tblgen_ftz; auto tblgen_rounding = getProperties().rounding; (void)tblgen_rounding; if (tblgen_rounding && !((::llvm::isa<::mlir::NVVM::FPRoundingModeAttr>(tblgen_rounding)))) return emitError(loc, "'nvgpu.rcp' op attribute 'rounding' failed to satisfy constraint: NVVM FPRoundingMode kind"); if (tblgen_approx && !((::llvm::isa<::mlir::BoolAttr>(tblgen_approx)))) return emitError(loc, "'nvgpu.rcp' op attribute 'approx' failed to satisfy constraint: bool attribute"); if (tblgen_ftz && !((::llvm::isa<::mlir::BoolAttr>(tblgen_ftz)))) return emitError(loc, "'nvgpu.rcp' op attribute 'ftz' failed to satisfy constraint: bool attribute"); return ::mlir::success(); } ::llvm::LogicalResult RcpOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.approx; auto attr = dict.get("approx"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `approx` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.ftz; auto attr = dict.get("ftz"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `ftz` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.rounding; auto attr = dict.get("rounding"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `rounding` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute RcpOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.approx; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("approx", propStorage)); } { const auto &propStorage = prop.ftz; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("ftz", propStorage)); } { const auto &propStorage = prop.rounding; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("rounding", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code RcpOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.approx.getAsOpaquePointer()), llvm::hash_value(prop.ftz.getAsOpaquePointer()), llvm::hash_value(prop.rounding.getAsOpaquePointer())); } std::optional RcpOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "approx") return prop.approx; if (name == "ftz") return prop.ftz; if (name == "rounding") return prop.rounding; return std::nullopt; } void RcpOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "approx") { prop.approx = ::llvm::dyn_cast_or_null>(value); return; } if (name == "ftz") { prop.ftz = ::llvm::dyn_cast_or_null>(value); return; } if (name == "rounding") { prop.rounding = ::llvm::dyn_cast_or_null>(value); return; } } void RcpOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.approx) attrs.append("approx", prop.approx); if (prop.ftz) attrs.append("ftz", prop.ftz); if (prop.rounding) attrs.append("rounding", prop.rounding); } ::llvm::LogicalResult RcpOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getApproxAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "approx", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getFtzAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "ftz", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getRoundingAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps6(attr, "rounding", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult RcpOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.approx))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.ftz))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.rounding))) return ::mlir::failure(); return ::mlir::success(); } void RcpOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.approx); writer.writeOptionalAttribute(prop.ftz); writer.writeOptionalAttribute(prop.rounding); } ::mlir::NVVM::FPRoundingMode RcpOp::getRounding() { auto attr = getRoundingAttr(); return attr.getValue(); } bool RcpOp::getApprox() { auto attr = getApproxAttr(); return attr.getValue(); } bool RcpOp::getFtz() { auto attr = getFtzAttr(); return attr.getValue(); } void RcpOp::setRounding(::mlir::NVVM::FPRoundingMode attrValue) { getProperties().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void RcpOp::setApprox(bool attrValue) { getProperties().approx = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue); } void RcpOp::setFtz(bool attrValue) { getProperties().ftz = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { odsState.addOperands(in); if (rounding) { odsState.getOrAddProperties().rounding = rounding; } if (approx) { odsState.getOrAddProperties().approx = approx; } if (ftz) { odsState.getOrAddProperties().ftz = ftz; } odsState.addTypes(out); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(out), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { return create(builder, builder.getLoc(), std::forward(out), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { odsState.addOperands(in); if (rounding) { odsState.getOrAddProperties().rounding = rounding; } if (approx) { odsState.getOrAddProperties().approx = approx; } if (ftz) { odsState.getOrAddProperties().ftz = ftz; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { return create(builder, builder.getLoc(), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { odsState.addOperands(in); if (rounding) { odsState.getOrAddProperties().rounding = rounding; } if (approx) { odsState.getOrAddProperties().approx = approx; } if (ftz) { odsState.getOrAddProperties().ftz = ftz; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { odsState.addOperands(in); odsState.getOrAddProperties().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding); odsState.getOrAddProperties().approx = odsBuilder.getBoolAttr(approx); odsState.getOrAddProperties().ftz = odsBuilder.getBoolAttr(ftz); odsState.addTypes(out); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(out), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { return create(builder, builder.getLoc(), std::forward(out), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { odsState.addOperands(in); odsState.getOrAddProperties().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding); odsState.getOrAddProperties().approx = odsBuilder.getBoolAttr(approx); odsState.getOrAddProperties().ftz = odsBuilder.getBoolAttr(ftz); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { return create(builder, builder.getLoc(), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { odsState.addOperands(in); odsState.getOrAddProperties().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding); odsState.getOrAddProperties().approx = odsBuilder.getBoolAttr(approx); odsState.getOrAddProperties().ftz = odsBuilder.getBoolAttr(ftz); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(in), std::forward(rounding), std::forward(approx), std::forward(ftz)); } void RcpOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void RcpOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void RcpOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.rounding) properties.rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), ::mlir::NVVM::FPRoundingMode::NONE); if (!properties.approx) properties.approx = odsBuilder.getBoolAttr(false); if (!properties.ftz) properties.ftz = odsBuilder.getBoolAttr(false); } ::llvm::LogicalResult RcpOp::verifyInvariantsImpl() { auto tblgen_approx = getProperties().approx; (void)tblgen_approx; auto tblgen_ftz = getProperties().ftz; (void)tblgen_ftz; auto tblgen_rounding = getProperties().rounding; (void)tblgen_rounding; if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps6(*this, tblgen_rounding, "rounding"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_approx, "approx"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_ftz, "ftz"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps12(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult RcpOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult RcpOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult RcpOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand inRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> inOperands(&inRawOperand, 1); ::llvm::SMLoc inOperandsLoc; (void)inOperandsLoc; ::mlir::Type outRawType{}; ::llvm::ArrayRef<::mlir::Type> outTypes(&outRawType, 1); inOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(inRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); outRawType = type; } result.addTypes(outTypes); if (parser.resolveOperands(inOperands, outTypes[0], inOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void RcpOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getIn(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getRoundingAttr(); if(attr && (attr == ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), ::mlir::NVVM::FPRoundingMode::NONE))) elidedAttrs.push_back("rounding"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getApproxAttr(); if(attr && (attr == odsBuilder.getBoolAttr(false))) elidedAttrs.push_back("approx"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getFtzAttr(); if(attr && (attr == odsBuilder.getBoolAttr(false))) elidedAttrs.push_back("ftz"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getOut().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void RcpOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::RcpOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::RcpOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::TmaAsyncLoadOp definitions //===----------------------------------------------------------------------===// namespace detail { TmaAsyncLoadOpGenericAdaptorBase::TmaAsyncLoadOpGenericAdaptorBase(TmaAsyncLoadOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TmaAsyncLoadOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail TmaAsyncLoadOpAdaptor::TmaAsyncLoadOpAdaptor(TmaAsyncLoadOp op) : TmaAsyncLoadOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TmaAsyncLoadOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair TmaAsyncLoadOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange TmaAsyncLoadOp::getCoordinatesMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TmaAsyncLoadOp::getMulticastMaskMutable() { auto range = getODSOperandIndexAndLength(5); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(5u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TmaAsyncLoadOp::getPredicateMutable() { auto range = getODSOperandIndexAndLength(6); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(6u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult TmaAsyncLoadOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TmaAsyncLoadOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TmaAsyncLoadOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional TmaAsyncLoadOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void TmaAsyncLoadOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void TmaAsyncLoadOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult TmaAsyncLoadOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult TmaAsyncLoadOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void TmaAsyncLoadOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void TmaAsyncLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { odsState.addOperands(dst); odsState.addOperands(barriers); odsState.addOperands(tensorMapDescriptor); odsState.addOperands(coordinates); odsState.addOperands(mbarId); if (multicastMask) odsState.addOperands(multicastMask); if (predicate) odsState.addOperands(predicate); ::llvm::copy(::llvm::ArrayRef({1, 1, 1, static_cast(coordinates.size()), 1, (multicastMask ? 1 : 0), (predicate ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dst), std::forward(barriers), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(mbarId), std::forward(multicastMask), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(dst), std::forward(barriers), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(mbarId), std::forward(multicastMask), std::forward(predicate)); } void TmaAsyncLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { odsState.addOperands(dst); odsState.addOperands(barriers); odsState.addOperands(tensorMapDescriptor); odsState.addOperands(coordinates); odsState.addOperands(mbarId); if (multicastMask) odsState.addOperands(multicastMask); if (predicate) odsState.addOperands(predicate); ::llvm::copy(::llvm::ArrayRef({1, 1, 1, static_cast(coordinates.size()), 1, (multicastMask ? 1 : 0), (predicate ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dst), std::forward(barriers), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(mbarId), std::forward(multicastMask), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dst), std::forward(barriers), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(mbarId), std::forward(multicastMask), std::forward(predicate)); } void TmaAsyncLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TmaAsyncLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TmaAsyncLoadOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); if (valueGroup5.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup5.size(); } for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); if (valueGroup6.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup6.size(); } for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult TmaAsyncLoadOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult TmaAsyncLoadOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc; (void)tensorMapDescriptorOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> coordinatesOperands; ::llvm::SMLoc coordinatesOperandsLoc; (void)coordinatesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc; (void)barriersOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc; (void)mbarIdOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc; (void)dstOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> multicastMaskOperands; ::llvm::SMLoc multicastMaskOperandsLoc; (void)multicastMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands; ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type tensorMapDescriptorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1); ::mlir::Type barriersRawType{}; ::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1); ::mlir::Type dstRawType{}; ::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1); tensorMapDescriptorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorMapDescriptorRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); coordinatesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(coordinatesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); barriersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(barriersRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); mbarIdOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mbarIdRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseKeyword("to")) return ::mlir::failure(); dstOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("multicast_mask"))) { if (parser.parseEqual()) return ::mlir::failure(); { multicastMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); multicastMaskOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("predicate")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); { predicateOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); predicateOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapDescriptorRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::nvgpu::MBarrierGroupType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); barriersRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, 1, static_cast(coordinatesOperands.size()), 1, static_cast(multicastMaskOperands.size()), static_cast(predicateOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(16); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIntegerType(1); if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(coordinatesOperands, odsBuildableType0, coordinatesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands)) return ::mlir::failure(); if (!multicastMaskOperands.empty()) { if (parser.resolveOperands(multicastMaskOperands, odsBuildableType1, multicastMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!predicateOperands.empty()) { if (parser.resolveOperands(predicateOperands, odsBuildableType2, predicateOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void TmaAsyncLoadOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensorMapDescriptor(); _odsPrinter << "["; _odsPrinter << getCoordinates(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBarriers(); _odsPrinter << "["; _odsPrinter << getMbarId(); _odsPrinter << "]"; _odsPrinter << ' ' << "to"; _odsPrinter << ' '; _odsPrinter << getDst(); if (getMulticastMask()) { _odsPrinter << ' ' << "multicast_mask"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getMulticastMask()) _odsPrinter << value; } if (getPredicate()) { _odsPrinter << ","; _odsPrinter << ' ' << "predicate"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getPredicate()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTensorMapDescriptor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getBarriers().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getDst().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void TmaAsyncLoadOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaAsyncLoadOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::TmaAsyncLoadOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::TmaAsyncStoreOp definitions //===----------------------------------------------------------------------===// namespace detail { TmaAsyncStoreOpGenericAdaptorBase::TmaAsyncStoreOpGenericAdaptorBase(TmaAsyncStoreOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TmaAsyncStoreOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail TmaAsyncStoreOpAdaptor::TmaAsyncStoreOpAdaptor(TmaAsyncStoreOp op) : TmaAsyncStoreOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TmaAsyncStoreOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair TmaAsyncStoreOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange TmaAsyncStoreOp::getCoordinatesMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TmaAsyncStoreOp::getPredicateMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult TmaAsyncStoreOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TmaAsyncStoreOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TmaAsyncStoreOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional TmaAsyncStoreOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void TmaAsyncStoreOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void TmaAsyncStoreOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult TmaAsyncStoreOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult TmaAsyncStoreOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void TmaAsyncStoreOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void TmaAsyncStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { odsState.addOperands(src); odsState.addOperands(tensorMapDescriptor); odsState.addOperands(coordinates); if (predicate) odsState.addOperands(predicate); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(coordinates.size()), (predicate ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(src), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(src), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(predicate)); } void TmaAsyncStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { odsState.addOperands(src); odsState.addOperands(tensorMapDescriptor); odsState.addOperands(coordinates); if (predicate) odsState.addOperands(predicate); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(coordinates.size()), (predicate ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(src), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(src), std::forward(tensorMapDescriptor), std::forward(coordinates), std::forward(predicate)); } void TmaAsyncStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TmaAsyncStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TmaAsyncStoreOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult TmaAsyncStoreOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult TmaAsyncStoreOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc; (void)srcOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc; (void)tensorMapDescriptorOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> coordinatesOperands; ::llvm::SMLoc coordinatesOperandsLoc; (void)coordinatesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands; ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type srcRawType{}; ::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1); ::mlir::Type tensorMapDescriptorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1); srcOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcRawOperand)) return ::mlir::failure(); if (parser.parseKeyword("to")) return ::mlir::failure(); tensorMapDescriptorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorMapDescriptorRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); coordinatesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(coordinatesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("predicate")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); { predicateOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); predicateOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapDescriptorRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(coordinatesOperands.size()), static_cast(predicateOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1); if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(coordinatesOperands, odsBuildableType0, coordinatesOperandsLoc, result.operands)) return ::mlir::failure(); if (!predicateOperands.empty()) { if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void TmaAsyncStoreOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrc(); _odsPrinter << ' ' << "to"; _odsPrinter << ' '; _odsPrinter << getTensorMapDescriptor(); _odsPrinter << "["; _odsPrinter << getCoordinates(); _odsPrinter << "]"; if (getPredicate()) { _odsPrinter << ","; _odsPrinter << ' ' << "predicate"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getPredicate()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSrc().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getTensorMapDescriptor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void TmaAsyncStoreOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaAsyncStoreOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::TmaAsyncStoreOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::TmaCreateDescriptorOp definitions //===----------------------------------------------------------------------===// namespace detail { TmaCreateDescriptorOpGenericAdaptorBase::TmaCreateDescriptorOpGenericAdaptorBase(TmaCreateDescriptorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TmaCreateDescriptorOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail TmaCreateDescriptorOpAdaptor::TmaCreateDescriptorOpAdaptor(TmaCreateDescriptorOp op) : TmaCreateDescriptorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TmaCreateDescriptorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair TmaCreateDescriptorOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange TmaCreateDescriptorOp::getBoxDimensionsMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult TmaCreateDescriptorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TmaCreateDescriptorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TmaCreateDescriptorOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional TmaCreateDescriptorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void TmaCreateDescriptorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void TmaCreateDescriptorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult TmaCreateDescriptorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void TmaCreateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { odsState.addOperands(tensor); odsState.addOperands(boxDimensions); odsState.addTypes(tensorMap); } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tensorMap), std::forward(tensor), std::forward(boxDimensions)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { return create(builder, builder.getLoc(), std::forward(tensorMap), std::forward(tensor), std::forward(boxDimensions)); } void TmaCreateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { odsState.addOperands(tensor); odsState.addOperands(boxDimensions); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tensor), std::forward(boxDimensions)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tensor), std::forward(boxDimensions)); } void TmaCreateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TmaCreateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TmaCreateDescriptorOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult TmaCreateDescriptorOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult TmaCreateDescriptorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tensorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorOperands(&tensorRawOperand, 1); ::llvm::SMLoc tensorOperandsLoc; (void)tensorOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> boxDimensionsOperands; ::llvm::SMLoc boxDimensionsOperandsLoc; (void)boxDimensionsOperandsLoc; ::mlir::Type tensorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorTypes(&tensorRawType, 1); ::mlir::Type tensorMapRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapTypes(&tensorMapRawType, 1); tensorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorRawOperand)) return ::mlir::failure(); if (parser.parseKeyword("box")) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); boxDimensionsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(boxDimensionsOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::UnrankedMemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(tensorMapTypes); if (parser.resolveOperands(tensorOperands, tensorTypes, tensorOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(boxDimensionsOperands, odsBuildableType0, boxDimensionsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void TmaCreateDescriptorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensor(); _odsPrinter << ' ' << "box"; _odsPrinter << "["; _odsPrinter << getBoxDimensions(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTensor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::UnrankedMemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getTensorMap().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaCreateDescriptorOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::TmaFenceOp definitions //===----------------------------------------------------------------------===// namespace detail { TmaFenceOpGenericAdaptorBase::TmaFenceOpGenericAdaptorBase(TmaFenceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail TmaFenceOpAdaptor::TmaFenceOpAdaptor(TmaFenceOp op) : TmaFenceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TmaFenceOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult TmaFenceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TmaFenceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TmaFenceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional TmaFenceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void TmaFenceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void TmaFenceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult TmaFenceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void TmaFenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tensorMapDescriptor) { odsState.addOperands(tensorMapDescriptor); } TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tensorMapDescriptor) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tensorMapDescriptor)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tensorMapDescriptor) { return create(builder, builder.getLoc(), std::forward(tensorMapDescriptor)); } void TmaFenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) { odsState.addOperands(tensorMapDescriptor); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tensorMapDescriptor)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tensorMapDescriptor)); } void TmaFenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TmaFenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TmaFenceOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult TmaFenceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult TmaFenceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc; (void)tensorMapDescriptorOperandsLoc; ::mlir::Type tensorMapDescriptorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1); tensorMapDescriptorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorMapDescriptorRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapDescriptorRawType = type; } if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void TmaFenceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensorMapDescriptor(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTensorMapDescriptor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaFenceOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::TmaPrefetchOp definitions //===----------------------------------------------------------------------===// namespace detail { TmaPrefetchOpGenericAdaptorBase::TmaPrefetchOpGenericAdaptorBase(TmaPrefetchOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TmaPrefetchOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail TmaPrefetchOpAdaptor::TmaPrefetchOpAdaptor(TmaPrefetchOp op) : TmaPrefetchOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TmaPrefetchOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair TmaPrefetchOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange TmaPrefetchOp::getPredicateMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult TmaPrefetchOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TmaPrefetchOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TmaPrefetchOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional TmaPrefetchOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void TmaPrefetchOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void TmaPrefetchOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult TmaPrefetchOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void TmaPrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { odsState.addOperands(tensorMapDescriptor); if (predicate) odsState.addOperands(predicate); } TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tensorMapDescriptor), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(tensorMapDescriptor), std::forward(predicate)); } void TmaPrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { odsState.addOperands(tensorMapDescriptor); if (predicate) odsState.addOperands(predicate); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tensorMapDescriptor), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tensorMapDescriptor), std::forward(predicate)); } void TmaPrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TmaPrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TmaPrefetchOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); if (valueGroup1.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult TmaPrefetchOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult TmaPrefetchOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc; (void)tensorMapDescriptorOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands; ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type tensorMapDescriptorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1); tensorMapDescriptorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorMapDescriptorRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("predicate")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); { predicateOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); predicateOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapDescriptorRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands)) return ::mlir::failure(); if (!predicateOperands.empty()) { if (parser.resolveOperands(predicateOperands, odsBuildableType0, predicateOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void TmaPrefetchOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensorMapDescriptor(); if (getPredicate()) { _odsPrinter << ","; _odsPrinter << ' ' << "predicate"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getPredicate()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTensorMapDescriptor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaPrefetchOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::WarpgroupGenerateDescriptorOp definitions //===----------------------------------------------------------------------===// namespace detail { WarpgroupGenerateDescriptorOpGenericAdaptorBase::WarpgroupGenerateDescriptorOpGenericAdaptorBase(WarpgroupGenerateDescriptorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail WarpgroupGenerateDescriptorOpAdaptor::WarpgroupGenerateDescriptorOpAdaptor(WarpgroupGenerateDescriptorOp op) : WarpgroupGenerateDescriptorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WarpgroupGenerateDescriptorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult WarpgroupGenerateDescriptorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute WarpgroupGenerateDescriptorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WarpgroupGenerateDescriptorOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional WarpgroupGenerateDescriptorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void WarpgroupGenerateDescriptorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void WarpgroupGenerateDescriptorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) { odsState.addOperands(tensor); odsState.addOperands(tensorMap); odsState.addTypes(descriptor); } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(descriptor), std::forward(tensor), std::forward(tensorMap)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) { return create(builder, builder.getLoc(), std::forward(descriptor), std::forward(tensor), std::forward(tensorMap)); } void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) { odsState.addOperands(tensor); odsState.addOperands(tensorMap); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tensor), std::forward(tensorMap)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tensor), std::forward(tensorMap)); } void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult WarpgroupGenerateDescriptorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tensorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorOperands(&tensorRawOperand, 1); ::llvm::SMLoc tensorOperandsLoc; (void)tensorOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tensorMapRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapOperands(&tensorMapRawOperand, 1); ::llvm::SMLoc tensorMapOperandsLoc; (void)tensorMapOperandsLoc; ::mlir::Type tensorRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorTypes(&tensorRawType, 1); ::mlir::Type tensorMapRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorMapTypes(&tensorMapRawType, 1); ::mlir::Type descriptorRawType{}; ::llvm::ArrayRef<::mlir::Type> descriptorTypes(&descriptorRawType, 1); tensorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tensorMapOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorMapRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::nvgpu::TensorMapDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tensorMapRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupMatrixDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); descriptorRawType = type; } result.addTypes(descriptorTypes); if (parser.resolveOperands(tensorOperands, tensorTypes, tensorOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tensorMapOperands, tensorMapTypes, tensorMapOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void WarpgroupGenerateDescriptorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensor(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTensorMap(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTensor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getTensorMap().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getDescriptor().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void WarpgroupGenerateDescriptorOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupGenerateDescriptorOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp definitions //===----------------------------------------------------------------------===// namespace detail { WarpgroupMmaInitAccumulatorOpGenericAdaptorBase::WarpgroupMmaInitAccumulatorOpGenericAdaptorBase(WarpgroupMmaInitAccumulatorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail WarpgroupMmaInitAccumulatorOpAdaptor::WarpgroupMmaInitAccumulatorOpAdaptor(WarpgroupMmaInitAccumulatorOp op) : WarpgroupMmaInitAccumulatorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WarpgroupMmaInitAccumulatorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute WarpgroupMmaInitAccumulatorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WarpgroupMmaInitAccumulatorOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional WarpgroupMmaInitAccumulatorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void WarpgroupMmaInitAccumulatorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void WarpgroupMmaInitAccumulatorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixC) { odsState.addTypes(matrixC); } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixC) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixC)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixC) { return create(builder, builder.getLoc(), std::forward(matrixC)); } void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult WarpgroupMmaInitAccumulatorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::Type matrixCRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupAccumulatorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixCRawType = type; } result.addTypes(matrixCTypes); return ::mlir::success(); } void WarpgroupMmaInitAccumulatorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getMatrixC().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::WarpgroupMmaOp definitions //===----------------------------------------------------------------------===// namespace detail { WarpgroupMmaOpGenericAdaptorBase::WarpgroupMmaOpGenericAdaptorBase(WarpgroupMmaOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::IntegerAttr WarpgroupMmaOpGenericAdaptorBase::getWaitGroupAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::IntegerAttr>(getProperties().waitGroup); if (!attr) attr = ::mlir::Builder(odsAttrs.getContext()).getIntegerAttr(::mlir::Builder(odsAttrs.getContext()).getIntegerType(64), 1); return attr; } uint64_t WarpgroupMmaOpGenericAdaptorBase::getWaitGroup() { auto attr = getWaitGroupAttr(); if (!attr) return ::mlir::Builder(odsAttrs.getContext()).getIntegerAttr(::mlir::Builder(odsAttrs.getContext()).getIntegerType(64), 1).getValue().getZExtValue(); return attr.getValue().getZExtValue(); } ::std::optional WarpgroupMmaOpGenericAdaptorBase::getTransposeA() { auto attr = getTransposeAAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional WarpgroupMmaOpGenericAdaptorBase::getTransposeB() { auto attr = getTransposeBAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail WarpgroupMmaOpAdaptor::WarpgroupMmaOpAdaptor(WarpgroupMmaOp op) : WarpgroupMmaOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WarpgroupMmaOpAdaptor::verify(::mlir::Location loc) { auto tblgen_transposeA = getProperties().transposeA; (void)tblgen_transposeA; auto tblgen_transposeB = getProperties().transposeB; (void)tblgen_transposeB; auto tblgen_waitGroup = getProperties().waitGroup; (void)tblgen_waitGroup; if (tblgen_waitGroup && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_waitGroup))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_waitGroup).getType().isSignlessInteger(64))))) return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'waitGroup' failed to satisfy constraint: 64-bit signless integer attribute"); if (tblgen_transposeA && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transposeA)))) return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'transposeA' failed to satisfy constraint: unit attribute"); if (tblgen_transposeB && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transposeB)))) return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'transposeB' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.transposeA; auto attr = dict.get("transposeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transposeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transposeB; auto attr = dict.get("transposeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transposeB` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.waitGroup; auto attr = dict.get("waitGroup"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `waitGroup` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute WarpgroupMmaOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.transposeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transposeA", propStorage)); } { const auto &propStorage = prop.transposeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transposeB", propStorage)); } { const auto &propStorage = prop.waitGroup; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("waitGroup", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WarpgroupMmaOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.transposeA.getAsOpaquePointer()), llvm::hash_value(prop.transposeB.getAsOpaquePointer()), llvm::hash_value(prop.waitGroup.getAsOpaquePointer())); } std::optional WarpgroupMmaOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "transposeA") return prop.transposeA; if (name == "transposeB") return prop.transposeB; if (name == "waitGroup") return prop.waitGroup; return std::nullopt; } void WarpgroupMmaOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "transposeA") { prop.transposeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "transposeB") { prop.transposeB = ::llvm::dyn_cast_or_null>(value); return; } if (name == "waitGroup") { prop.waitGroup = ::llvm::dyn_cast_or_null>(value); return; } } void WarpgroupMmaOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.transposeA) attrs.append("transposeA", prop.transposeA); if (prop.transposeB) attrs.append("transposeB", prop.transposeB); if (prop.waitGroup) attrs.append("waitGroup", prop.waitGroup); } ::llvm::LogicalResult WarpgroupMmaOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getTransposeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "transposeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTransposeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "transposeB", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getWaitGroupAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps7(attr, "waitGroup", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.transposeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.transposeB))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.waitGroup))) return ::mlir::failure(); return ::mlir::success(); } void WarpgroupMmaOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.transposeA); writer.writeOptionalAttribute(prop.transposeB); writer.writeOptionalAttribute(prop.waitGroup); } uint64_t WarpgroupMmaOp::getWaitGroup() { auto attr = getWaitGroupAttr(); if (!attr) return ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), 1).getValue().getZExtValue(); return attr.getValue().getZExtValue(); } ::std::optional WarpgroupMmaOp::getTransposeA() { auto attr = getTransposeAAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional WarpgroupMmaOp::getTransposeB() { auto attr = getTransposeBAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void WarpgroupMmaOp::setWaitGroup(::std::optional attrValue) { auto &odsProp = getProperties().waitGroup; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } void WarpgroupMmaOp::setTransposeA(bool attrValue) { auto &odsProp = getProperties().transposeA; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void WarpgroupMmaOp::setTransposeB(bool attrValue) { auto &odsProp = getProperties().transposeB; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { odsState.addOperands(descriptorA); odsState.addOperands(descriptorB); odsState.addOperands(matrixC); if (waitGroup) { odsState.getOrAddProperties().waitGroup = waitGroup; } if (transposeA) { odsState.getOrAddProperties().transposeA = transposeA; } if (transposeB) { odsState.getOrAddProperties().transposeB = transposeB; } odsState.addTypes(matrixD); } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixD), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { return create(builder, builder.getLoc(), std::forward(matrixD), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); } void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { odsState.addOperands(descriptorA); odsState.addOperands(descriptorB); odsState.addOperands(matrixC); if (waitGroup) { odsState.getOrAddProperties().waitGroup = waitGroup; } if (transposeA) { odsState.getOrAddProperties().transposeA = transposeA; } if (transposeB) { odsState.getOrAddProperties().transposeB = transposeB; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); } void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { odsState.addOperands(descriptorA); odsState.addOperands(descriptorB); odsState.addOperands(matrixC); odsState.getOrAddProperties().waitGroup = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), waitGroup); if (transposeA) { odsState.getOrAddProperties().transposeA = transposeA; } if (transposeB) { odsState.getOrAddProperties().transposeB = transposeB; } odsState.addTypes(matrixD); } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixD), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { return create(builder, builder.getLoc(), std::forward(matrixD), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); } void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { odsState.addOperands(descriptorA); odsState.addOperands(descriptorB); odsState.addOperands(matrixC); odsState.getOrAddProperties().waitGroup = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), waitGroup); if (transposeA) { odsState.getOrAddProperties().transposeA = transposeA; } if (transposeB) { odsState.getOrAddProperties().transposeB = transposeB; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(descriptorA), std::forward(descriptorB), std::forward(waitGroup), std::forward(transposeA), std::forward(transposeB), std::forward(matrixC)); } void WarpgroupMmaOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void WarpgroupMmaOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult WarpgroupMmaOp::verifyInvariantsImpl() { auto tblgen_transposeA = getProperties().transposeA; (void)tblgen_transposeA; auto tblgen_transposeB = getProperties().transposeB; (void)tblgen_transposeB; auto tblgen_waitGroup = getProperties().waitGroup; (void)tblgen_waitGroup; if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps7(*this, tblgen_waitGroup, "waitGroup"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_transposeA, "transposeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_transposeB, "transposeB"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult WarpgroupMmaOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand descriptorARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descriptorAOperands(&descriptorARawOperand, 1); ::llvm::SMLoc descriptorAOperandsLoc; (void)descriptorAOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand descriptorBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descriptorBOperands(&descriptorBRawOperand, 1); ::llvm::SMLoc descriptorBOperandsLoc; (void)descriptorBOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc; (void)matrixCOperandsLoc; ::mlir::Type descriptorARawType{}; ::llvm::ArrayRef<::mlir::Type> descriptorATypes(&descriptorARawType, 1); ::mlir::Type descriptorBRawType{}; ::llvm::ArrayRef<::mlir::Type> descriptorBTypes(&descriptorBRawType, 1); ::mlir::Type matrixCRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1); ::mlir::Type matrixDRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixDTypes(&matrixDRawType, 1); descriptorAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(descriptorARawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); descriptorBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(descriptorBRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixCRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupMatrixDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); descriptorARawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupMatrixDescriptorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); descriptorBRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupAccumulatorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixCRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupAccumulatorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixDRawType = type; } result.addTypes(matrixDTypes); if (parser.resolveOperands(descriptorAOperands, descriptorATypes, descriptorAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(descriptorBOperands, descriptorBTypes, descriptorBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void WarpgroupMmaOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getDescriptorA(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDescriptorB(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrixC(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getWaitGroupAttr(); if(attr && (attr == odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), 1))) elidedAttrs.push_back("waitGroup"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getDescriptorA().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getDescriptorB().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrixC().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getMatrixD().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::WarpgroupMmaOpGenericAdaptorBase::Properties) namespace mlir::nvgpu { //===----------------------------------------------------------------------===// // ::mlir::nvgpu::WarpgroupMmaStoreOp definitions //===----------------------------------------------------------------------===// namespace detail { WarpgroupMmaStoreOpGenericAdaptorBase::WarpgroupMmaStoreOpGenericAdaptorBase(WarpgroupMmaStoreOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail WarpgroupMmaStoreOpAdaptor::WarpgroupMmaStoreOpAdaptor(WarpgroupMmaStoreOp op) : WarpgroupMmaStoreOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WarpgroupMmaStoreOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaStoreOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute WarpgroupMmaStoreOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WarpgroupMmaStoreOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional WarpgroupMmaStoreOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void WarpgroupMmaStoreOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void WarpgroupMmaStoreOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value matrixD, ::mlir::Value dstMemref) { odsState.addOperands(matrixD); odsState.addOperands(dstMemref); } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value matrixD, ::mlir::Value dstMemref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrixD), std::forward(dstMemref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value matrixD, ::mlir::Value dstMemref) { return create(builder, builder.getLoc(), std::forward(matrixD), std::forward(dstMemref)); } void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) { odsState.addOperands(matrixD); odsState.addOperands(dstMemref); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(matrixD), std::forward(dstMemref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(matrixD), std::forward(dstMemref)); } void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult WarpgroupMmaStoreOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand matrixDRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixDOperands(&matrixDRawOperand, 1); ::llvm::SMLoc matrixDOperandsLoc; (void)matrixDOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstMemrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstMemrefOperands(&dstMemrefRawOperand, 1); ::llvm::SMLoc dstMemrefOperandsLoc; (void)dstMemrefOperandsLoc; ::mlir::Type matrixDRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixDTypes(&matrixDRawType, 1); ::mlir::Type dstMemrefRawType{}; ::llvm::ArrayRef<::mlir::Type> dstMemrefTypes(&dstMemrefRawType, 1); matrixDOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixDRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dstMemrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstMemrefRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::nvgpu::WarpgroupAccumulatorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixDRawType = type; } if (parser.parseKeyword("to")) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstMemrefRawType = type; } if (parser.resolveOperands(matrixDOperands, matrixDTypes, matrixDOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dstMemrefOperands, dstMemrefTypes, dstMemrefOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void WarpgroupMmaStoreOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getMatrixD(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDstMemref(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMatrixD().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "to"; _odsPrinter << ' '; { auto type = getDstMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void WarpgroupMmaStoreOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::nvgpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaStoreOp) #endif // GET_OP_CLASSES