9776 lines
473 KiB
C++
9776 lines
473 KiB
C++
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
|
|
|* *|
|
|
|* Op Definitions *|
|
|
|* *|
|
|
|* Automatically generated file, do not edit! *|
|
|
|* From: NVGPUOps.td *|
|
|
|* *|
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
#ifdef GET_OP_LIST
|
|
#undef GET_OP_LIST
|
|
|
|
::mlir::nvgpu::DeviceAsyncCopyOp,
|
|
::mlir::nvgpu::DeviceAsyncCreateGroupOp,
|
|
::mlir::nvgpu::DeviceAsyncWaitOp,
|
|
::mlir::nvgpu::LdMatrixOp,
|
|
::mlir::nvgpu::MBarrierArriveExpectTxOp,
|
|
::mlir::nvgpu::MBarrierArriveNoCompleteOp,
|
|
::mlir::nvgpu::MBarrierArriveOp,
|
|
::mlir::nvgpu::MBarrierCreateOp,
|
|
::mlir::nvgpu::MBarrierGetOp,
|
|
::mlir::nvgpu::MBarrierInitOp,
|
|
::mlir::nvgpu::MBarrierTestWaitOp,
|
|
::mlir::nvgpu::MBarrierTryWaitParityOp,
|
|
::mlir::nvgpu::MmaSparseSyncOp,
|
|
::mlir::nvgpu::MmaSyncOp,
|
|
::mlir::nvgpu::RcpOp,
|
|
::mlir::nvgpu::TmaAsyncLoadOp,
|
|
::mlir::nvgpu::TmaAsyncStoreOp,
|
|
::mlir::nvgpu::TmaCreateDescriptorOp,
|
|
::mlir::nvgpu::TmaFenceOp,
|
|
::mlir::nvgpu::TmaPrefetchOp,
|
|
::mlir::nvgpu::WarpgroupGenerateDescriptorOp,
|
|
::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp,
|
|
::mlir::nvgpu::WarpgroupMmaOp,
|
|
::mlir::nvgpu::WarpgroupMmaStoreOp
|
|
#endif // GET_OP_LIST
|
|
|
|
#ifdef GET_OP_CLASSES
|
|
#undef GET_OP_CLASSES
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Local Utility Method Definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps1(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be memref of any non-token type values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps2(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::IndexType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be variadic of index, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps3(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::IndexType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be index, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps4(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::DeviceAsyncTokenType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be device async token type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps5(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::DeviceAsyncTokenType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be variadic of device async token type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps6(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be vector of any non-token type values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps7(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::MBarrierGroupType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be mbarrier barrier type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps8(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((type.isSignlessInteger(1)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be 1-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps9(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::MBarrierTokenType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be , but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps10(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be 32-bit signless integer or 64-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps11(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements()
|
|
== 2))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of 16-bit signless integer values of length 2, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps12(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be vector of 32-bit float values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps13(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::TensorMapDescriptorType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be TensorMap descriptor, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps14(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((type.isSignlessInteger(16)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be 16-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps15(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((::llvm::isa<::mlir::UnrankedMemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be unranked.memref of any non-token type values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps16(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be Warpgroup matrix descriptor type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_NVGPUOps17(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::nvgpu::WarpgroupAccumulatorType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be , but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps1(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(attr).getType())))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: index attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps1(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps1(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps2(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::UnitAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: unit attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps2(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps2(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps3(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(32)))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: 32-bit signless integer attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps3(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps3(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps4(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::BoolAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: bool attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps4(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps4(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps5(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !(((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); }))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: 64-bit integer array attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps5(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps5(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps6(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::NVVM::FPRoundingModeAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: NVVM FPRoundingMode kind";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps6(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps6(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps7(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: 64-bit signless integer attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_NVGPUOps7(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_NVGPUOps7(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::DeviceAsyncCopyOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
DeviceAsyncCopyOpGenericAdaptorBase::DeviceAsyncCopyOpGenericAdaptorBase(DeviceAsyncCopyOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> DeviceAsyncCopyOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::llvm::APInt DeviceAsyncCopyOpGenericAdaptorBase::getDstElements() {
|
|
auto attr = getDstElementsAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::std::optional<bool> DeviceAsyncCopyOpGenericAdaptorBase::getBypassL1() {
|
|
auto attr = getBypassL1Attr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
DeviceAsyncCopyOpAdaptor::DeviceAsyncCopyOpAdaptor(DeviceAsyncCopyOp op) : DeviceAsyncCopyOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_bypassL1 = getProperties().bypassL1; (void)tblgen_bypassL1;
|
|
auto tblgen_dstElements = getProperties().dstElements; (void)tblgen_dstElements;
|
|
if (!tblgen_dstElements) return emitError(loc, "'nvgpu.device_async_copy' op requires attribute 'dstElements'");
|
|
|
|
if (tblgen_dstElements && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_dstElements))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_dstElements).getType())))))
|
|
return emitError(loc, "'nvgpu.device_async_copy' op attribute 'dstElements' failed to satisfy constraint: index attribute");
|
|
|
|
if (tblgen_bypassL1 && !((::llvm::isa<::mlir::UnitAttr>(tblgen_bypassL1))))
|
|
return emitError(loc, "'nvgpu.device_async_copy' op attribute 'bypassL1' failed to satisfy constraint: unit attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> DeviceAsyncCopyOp::getODSOperandIndexAndLength(unsigned index) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::mlir::MutableOperandRange DeviceAsyncCopyOp::getDstIndicesMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange DeviceAsyncCopyOp::getSrcIndicesMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange DeviceAsyncCopyOp::getSrcElementsMutable() {
|
|
auto range = getODSOperandIndexAndLength(4);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.bypassL1;
|
|
auto attr = dict.get("bypassL1");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `bypassL1` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.dstElements;
|
|
auto attr = dict.get("dstElements");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `dstElements` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
|
|
::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {
|
|
return convertFromAttribute(propStorage, propAttr, emitError);
|
|
};
|
|
auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");;
|
|
;
|
|
if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() {
|
|
return emitError() << "for `operandSegmentSizes`: ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute DeviceAsyncCopyOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.bypassL1;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("bypassL1",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.dstElements;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("dstElements",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.operandSegmentSizes;
|
|
auto attr = [&]() -> ::mlir::Attribute {
|
|
return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage);
|
|
}();
|
|
attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code DeviceAsyncCopyOp::computePropertiesHash(const Properties &prop) {
|
|
auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code {
|
|
using ::llvm::hash_value;
|
|
return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));;
|
|
};
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.bypassL1.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.dstElements.getAsOpaquePointer()),
|
|
hash_operandSegmentSizes_(prop.operandSegmentSizes));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> DeviceAsyncCopyOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "bypassL1")
|
|
return prop.bypassL1;
|
|
|
|
if (name == "dstElements")
|
|
return prop.dstElements;
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }();
|
|
return std::nullopt;
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "bypassL1") {
|
|
prop.bypassL1 = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.bypassL1)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "dstElements") {
|
|
prop.dstElements = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.dstElements)>>(value);
|
|
return;
|
|
}
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") {
|
|
auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value);
|
|
if (!arrAttr) return;
|
|
if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t))
|
|
return;
|
|
llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin());
|
|
return;
|
|
}
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.bypassL1) attrs.append("bypassL1", prop.bypassL1);
|
|
|
|
if (prop.dstElements) attrs.append("dstElements", prop.dstElements);
|
|
attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }());
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getBypassL1AttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "bypassL1", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getDstElementsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps1(attr, "dstElements", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.bypassL1)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readAttribute(prop.dstElements)))
|
|
return ::mlir::failure();
|
|
|
|
if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
::mlir::DenseI32ArrayAttr attr;
|
|
if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure();
|
|
if (attr.size() > static_cast<int64_t>(sizeof(propStorage) / sizeof(int32_t))) {
|
|
reader.emitError("size mismatch for operand/result_segment_size");
|
|
return ::mlir::failure();
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>(attr), propStorage.begin());
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
auto readProp = [&]() {
|
|
|
|
if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
return reader.readSparseArray(::llvm::MutableArrayRef(propStorage));
|
|
;
|
|
return ::mlir::success();
|
|
};
|
|
if (::mlir::failed(readProp()))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.bypassL1);
|
|
writer.writeAttribute(prop.dstElements);
|
|
|
|
if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage));
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
|
|
if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
writer.writeSparseArray(::llvm::ArrayRef(propStorage));
|
|
;
|
|
}
|
|
}
|
|
|
|
::llvm::APInt DeviceAsyncCopyOp::getDstElements() {
|
|
auto attr = getDstElementsAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::std::optional<bool> DeviceAsyncCopyOp::getBypassL1() {
|
|
auto attr = getBypassL1Attr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::setDstElements(::llvm::APInt attrValue) {
|
|
getProperties().dstElements = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), attrValue);
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::setBypassL1(bool attrValue) {
|
|
auto &odsProp = getProperties().bypassL1;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = dstElements;
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
odsState.addTypes(asyncToken);
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = dstElements;
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = dstElements;
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::mlir::IntegerAttr dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements);
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
odsState.addTypes(asyncToken);
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements);
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(dstIndices);
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(srcIndices);
|
|
if (srcElements)
|
|
odsState.addOperands(srcElements);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndices.size()), 1, static_cast<int32_t>(srcIndices.size()), (srcElements ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
odsState.getOrAddProperties<Properties>().dstElements = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), dstElements);
|
|
if (bypassL1) {
|
|
odsState.getOrAddProperties<Properties>().bypassL1 = bypassL1;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::ValueRange dstIndices, ::mlir::Value src, ::mlir::ValueRange srcIndices, ::llvm::APInt dstElements, /*optional*/::mlir::Value srcElements, /*optional*/::mlir::UnitAttr bypassL1) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(dstIndices)>(dstIndices), std::forward<decltype(src)>(src), std::forward<decltype(srcIndices)>(srcIndices), std::forward<decltype(dstElements)>(dstElements), std::forward<decltype(srcElements)>(srcElements), std::forward<decltype(bypassL1)>(bypassL1));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<DeviceAsyncCopyOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<DeviceAsyncCopyOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCopyOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCopyOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCopyOp DeviceAsyncCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::verifyInvariantsImpl() {
|
|
auto tblgen_bypassL1 = getProperties().bypassL1; (void)tblgen_bypassL1;
|
|
auto tblgen_dstElements = getProperties().dstElements; (void)tblgen_dstElements;
|
|
if (!tblgen_dstElements) return emitOpError("requires attribute 'dstElements'");
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps1(*this, tblgen_dstElements, "dstElements")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_bypassL1, "bypassL1")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup4 = getODSOperands(4);
|
|
|
|
if (valueGroup4.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup4.size();
|
|
}
|
|
|
|
for (auto v : valueGroup4) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCopyOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::DeviceAsyncTokenType>();
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult DeviceAsyncCopyOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc;
|
|
(void)srcOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> srcIndicesOperands;
|
|
::llvm::SMLoc srcIndicesOperandsLoc;
|
|
(void)srcIndicesOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc;
|
|
(void)dstOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> dstIndicesOperands;
|
|
::llvm::SMLoc dstIndicesOperandsLoc;
|
|
(void)dstIndicesOperandsLoc;
|
|
::mlir::IntegerAttr dstElementsAttr;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> srcElementsOperands;
|
|
::llvm::SMLoc srcElementsOperandsLoc;
|
|
(void)srcElementsOperandsLoc;
|
|
::mlir::Type srcRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1);
|
|
::mlir::Type dstRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1);
|
|
|
|
srcOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(srcRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
srcIndicesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(srcIndicesOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
dstOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(dstRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
dstIndicesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(dstIndicesOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseCustomAttributeWithFallback(dstElementsAttr, parser.getBuilder().getIndexType())) {
|
|
return ::mlir::failure();
|
|
}
|
|
if (dstElementsAttr) result.getOrAddProperties<DeviceAsyncCopyOp::Properties>().dstElements = dstElementsAttr;
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
|
|
{
|
|
srcElementsOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
srcElementsOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
srcRawType = type;
|
|
}
|
|
if (parser.parseKeyword("to"))
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
dstRawType = type;
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(dstIndicesOperands.size()), 1, static_cast<int32_t>(srcIndicesOperands.size()), static_cast<int32_t>(srcElementsOperands.size())}), result.getOrAddProperties<DeviceAsyncCopyOp::Properties>().operandSegmentSizes.begin());
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>();
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType();
|
|
result.addTypes(odsBuildableType0);
|
|
if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(dstIndicesOperands, odsBuildableType1, dstIndicesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(srcIndicesOperands, odsBuildableType1, srcIndicesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!srcElementsOperands.empty()) {
|
|
if (parser.resolveOperands(srcElementsOperands, odsBuildableType1, srcElementsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSrc();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getSrcIndices();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDst();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getDstIndices();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter.printAttributeWithoutType(getDstElementsAttr());
|
|
if (getSrcElements()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getSrcElements())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("operandSegmentSizes");
|
|
elidedAttrs.push_back("dstElements");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSrc().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "to";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDst().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void DeviceAsyncCopyOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(0);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(2);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncCopyOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::DeviceAsyncCopyOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::DeviceAsyncCreateGroupOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
DeviceAsyncCreateGroupOpGenericAdaptorBase::DeviceAsyncCreateGroupOpGenericAdaptorBase(DeviceAsyncCreateGroupOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> DeviceAsyncCreateGroupOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 0) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
DeviceAsyncCreateGroupOpAdaptor::DeviceAsyncCreateGroupOpAdaptor(DeviceAsyncCreateGroupOp op) : DeviceAsyncCreateGroupOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> DeviceAsyncCreateGroupOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 0) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange DeviceAsyncCreateGroupOp::getInputTokensMutable() {
|
|
auto range = getODSOperandIndexAndLength(0);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute DeviceAsyncCreateGroupOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code DeviceAsyncCreateGroupOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> DeviceAsyncCreateGroupOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) {
|
|
odsState.addOperands(inputTokens);
|
|
odsState.addTypes(asyncToken);
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(inputTokens)>(inputTokens));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCreateGroupOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type asyncToken, ::mlir::ValueRange inputTokens) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(asyncToken)>(asyncToken), std::forward<decltype(inputTokens)>(inputTokens));
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCreateGroupOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCreateGroupOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCreateGroupOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCreateGroupOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(DeviceAsyncCreateGroupOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncCreateGroupOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncCreateGroupOp DeviceAsyncCreateGroupOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps5(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncCreateGroupOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::DeviceAsyncTokenType>();
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult DeviceAsyncCreateGroupOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> inputTokensOperands;
|
|
::llvm::SMLoc inputTokensOperandsLoc;
|
|
(void)inputTokensOperandsLoc;
|
|
|
|
inputTokensOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(inputTokensOperands))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>();
|
|
result.addTypes(odsBuildableType0);
|
|
if (parser.resolveOperands(inputTokensOperands, odsBuildableType0, inputTokensOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncCreateGroupOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getInputTokens();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncCreateGroupOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::DeviceAsyncWaitOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
DeviceAsyncWaitOpGenericAdaptorBase::DeviceAsyncWaitOpGenericAdaptorBase(DeviceAsyncWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::std::optional<uint32_t> DeviceAsyncWaitOpGenericAdaptorBase::getNumGroups() {
|
|
auto attr = getNumGroupsAttr();
|
|
return attr ? ::std::optional<uint32_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
DeviceAsyncWaitOpAdaptor::DeviceAsyncWaitOpAdaptor(DeviceAsyncWaitOp op) : DeviceAsyncWaitOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_numGroups = getProperties().numGroups; (void)tblgen_numGroups;
|
|
|
|
if (tblgen_numGroups && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_numGroups))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_numGroups).getType().isSignlessInteger(32)))))
|
|
return emitError(loc, "'nvgpu.device_async_wait' op attribute 'numGroups' failed to satisfy constraint: 32-bit signless integer attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.numGroups;
|
|
auto attr = dict.get("numGroups");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `numGroups` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute DeviceAsyncWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.numGroups;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("numGroups",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code DeviceAsyncWaitOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.numGroups.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> DeviceAsyncWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "numGroups")
|
|
return prop.numGroups;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "numGroups") {
|
|
prop.numGroups = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.numGroups)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.numGroups) attrs.append("numGroups", prop.numGroups);
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getNumGroupsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "numGroups", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.numGroups)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.numGroups);
|
|
}
|
|
|
|
::std::optional<uint32_t> DeviceAsyncWaitOp::getNumGroups() {
|
|
auto attr = getNumGroupsAttr();
|
|
return attr ? ::std::optional<uint32_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::setNumGroups(::std::optional<uint32_t> attrValue) {
|
|
auto &odsProp = getProperties().numGroups;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
odsState.addOperands(asyncDependencies);
|
|
if (numGroups) {
|
|
odsState.getOrAddProperties<Properties>().numGroups = numGroups;
|
|
}
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(asyncDependencies)>(asyncDependencies), std::forward<decltype(numGroups)>(numGroups));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(asyncDependencies)>(asyncDependencies), std::forward<decltype(numGroups)>(numGroups));
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
odsState.addOperands(asyncDependencies);
|
|
if (numGroups) {
|
|
odsState.getOrAddProperties<Properties>().numGroups = numGroups;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(asyncDependencies)>(asyncDependencies), std::forward<decltype(numGroups)>(numGroups));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value asyncDependencies, /*optional*/::mlir::IntegerAttr numGroups) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(asyncDependencies)>(asyncDependencies), std::forward<decltype(numGroups)>(numGroups));
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<DeviceAsyncWaitOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DeviceAsyncWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DeviceAsyncWaitOp DeviceAsyncWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOp::verifyInvariantsImpl() {
|
|
auto tblgen_numGroups = getProperties().numGroups; (void)tblgen_numGroups;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_numGroups, "numGroups")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps4(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DeviceAsyncWaitOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult DeviceAsyncWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand asyncDependenciesRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> asyncDependenciesOperands(&asyncDependenciesRawOperand, 1); ::llvm::SMLoc asyncDependenciesOperandsLoc;
|
|
(void)asyncDependenciesOperandsLoc;
|
|
|
|
asyncDependenciesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(asyncDependenciesRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getType<::mlir::nvgpu::DeviceAsyncTokenType>();
|
|
if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DeviceAsyncWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getAsyncDependencies();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::DeviceAsyncWaitOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::DeviceAsyncWaitOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::LdMatrixOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
LdMatrixOpGenericAdaptorBase::LdMatrixOpGenericAdaptorBase(LdMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> LdMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
bool LdMatrixOpGenericAdaptorBase::getTranspose() {
|
|
auto attr = getTransposeAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
uint32_t LdMatrixOpGenericAdaptorBase::getNumTiles() {
|
|
auto attr = getNumTilesAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
LdMatrixOpAdaptor::LdMatrixOpAdaptor(LdMatrixOp op) : LdMatrixOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult LdMatrixOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_numTiles = getProperties().numTiles; (void)tblgen_numTiles;
|
|
if (!tblgen_numTiles) return emitError(loc, "'nvgpu.ldmatrix' op requires attribute 'numTiles'");
|
|
auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose;
|
|
if (!tblgen_transpose) return emitError(loc, "'nvgpu.ldmatrix' op requires attribute 'transpose'");
|
|
|
|
if (tblgen_transpose && !((::llvm::isa<::mlir::BoolAttr>(tblgen_transpose))))
|
|
return emitError(loc, "'nvgpu.ldmatrix' op attribute 'transpose' failed to satisfy constraint: bool attribute");
|
|
|
|
if (tblgen_numTiles && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_numTiles))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_numTiles).getType().isSignlessInteger(32)))))
|
|
return emitError(loc, "'nvgpu.ldmatrix' op attribute 'numTiles' failed to satisfy constraint: 32-bit signless integer attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> LdMatrixOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange LdMatrixOp::getIndicesMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult LdMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.numTiles;
|
|
auto attr = dict.get("numTiles");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `numTiles` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.transpose;
|
|
auto attr = dict.get("transpose");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `transpose` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute LdMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.numTiles;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("numTiles",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.transpose;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("transpose",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code LdMatrixOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.numTiles.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.transpose.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> LdMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "numTiles")
|
|
return prop.numTiles;
|
|
|
|
if (name == "transpose")
|
|
return prop.transpose;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void LdMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "numTiles") {
|
|
prop.numTiles = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.numTiles)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "transpose") {
|
|
prop.transpose = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.transpose)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void LdMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.numTiles) attrs.append("numTiles", prop.numTiles);
|
|
|
|
if (prop.transpose) attrs.append("transpose", prop.transpose);
|
|
}
|
|
|
|
::llvm::LogicalResult LdMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getNumTilesAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "numTiles", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "transpose", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LdMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.numTiles)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readAttribute(prop.transpose)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LdMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.numTiles);
|
|
writer.writeAttribute(prop.transpose);
|
|
}
|
|
|
|
bool LdMatrixOp::getTranspose() {
|
|
auto attr = getTransposeAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
uint32_t LdMatrixOp::getNumTiles() {
|
|
auto attr = getNumTilesAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
void LdMatrixOp::setTranspose(bool attrValue) {
|
|
getProperties().transpose = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue);
|
|
}
|
|
|
|
void LdMatrixOp::setNumTiles(uint32_t attrValue) {
|
|
getProperties().numTiles = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue);
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
odsState.addOperands(srcMemref);
|
|
odsState.addOperands(indices);
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
odsState.getOrAddProperties<Properties>().numTiles = numTiles;
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
odsState.addOperands(srcMemref);
|
|
odsState.addOperands(indices);
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
odsState.getOrAddProperties<Properties>().numTiles = numTiles;
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::BoolAttr transpose, ::mlir::IntegerAttr numTiles) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
odsState.addOperands(srcMemref);
|
|
odsState.addOperands(indices);
|
|
odsState.getOrAddProperties<Properties>().transpose = odsBuilder.getBoolAttr(transpose);
|
|
odsState.getOrAddProperties<Properties>().numTiles = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), numTiles);
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
odsState.addOperands(srcMemref);
|
|
odsState.addOperands(indices);
|
|
odsState.getOrAddProperties<Properties>().transpose = odsBuilder.getBoolAttr(transpose);
|
|
odsState.getOrAddProperties<Properties>().numTiles = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), numTiles);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, bool transpose, uint32_t numTiles) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(srcMemref)>(srcMemref), std::forward<decltype(indices)>(indices), std::forward<decltype(transpose)>(transpose), std::forward<decltype(numTiles)>(numTiles));
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<LdMatrixOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void LdMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<LdMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LdMatrixOp LdMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult LdMatrixOp::verifyInvariantsImpl() {
|
|
auto tblgen_numTiles = getProperties().numTiles; (void)tblgen_numTiles;
|
|
if (!tblgen_numTiles) return emitOpError("requires attribute 'numTiles'");
|
|
auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose;
|
|
if (!tblgen_transpose) return emitOpError("requires attribute 'transpose'");
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_transpose, "transpose")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_numTiles, "numTiles")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((((*this->getOperation()).getNumResults() > 0)) && (((*this->getOperation()).getNumOperands() > 0)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getResult(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType())))) && ((getElementTypeOrSelf((*this->getOperation()).getResult(0)) == getElementTypeOrSelf((*this->getOperation()).getOperand(0))))))
|
|
return emitOpError("failed to verify that srcMemref and res have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LdMatrixOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult LdMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand srcMemrefRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcMemrefOperands(&srcMemrefRawOperand, 1); ::llvm::SMLoc srcMemrefOperandsLoc;
|
|
(void)srcMemrefOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands;
|
|
::llvm::SMLoc indicesOperandsLoc;
|
|
(void)indicesOperandsLoc;
|
|
::mlir::Type srcMemrefRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> srcMemrefTypes(&srcMemrefRawType, 1);
|
|
::mlir::Type resRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1);
|
|
|
|
srcMemrefOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(srcMemrefRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
indicesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(indicesOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
srcMemrefRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
resRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(resTypes);
|
|
if (parser.resolveOperands(srcMemrefOperands, srcMemrefTypes, srcMemrefOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LdMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSrcMemref();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getIndices();
|
|
_odsPrinter << "]";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSrcMemref().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getRes().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void LdMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(0);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::LdMatrixOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::LdMatrixOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierArriveExpectTxOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierArriveExpectTxOpGenericAdaptorBase::MBarrierArriveExpectTxOpGenericAdaptorBase(MBarrierArriveExpectTxOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> MBarrierArriveExpectTxOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 3) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierArriveExpectTxOpAdaptor::MBarrierArriveExpectTxOpAdaptor(MBarrierArriveExpectTxOp op) : MBarrierArriveExpectTxOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierArriveExpectTxOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> MBarrierArriveExpectTxOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 3) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange MBarrierArriveExpectTxOp::getPredicateMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveExpectTxOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierArriveExpectTxOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierArriveExpectTxOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierArriveExpectTxOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(txcount);
|
|
odsState.addOperands(mbarId);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(txcount)>(txcount), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveExpectTxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(txcount)>(txcount), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(txcount);
|
|
odsState.addOperands(mbarId);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(txcount)>(txcount), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveExpectTxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value txcount, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(txcount)>(txcount), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveExpectTxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveExpectTxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveExpectTxOp MBarrierArriveExpectTxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
if (valueGroup3.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup3.size();
|
|
}
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveExpectTxOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierArriveExpectTxOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand txcountRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> txcountOperands(&txcountRawOperand, 1); ::llvm::SMLoc txcountOperandsLoc;
|
|
(void)txcountOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands;
|
|
::llvm::SMLoc predicateOperandsLoc;
|
|
(void)predicateOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
txcountOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(txcountRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("predicate"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
predicateOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
predicateOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(txcountOperands, odsBuildableType0, txcountOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!predicateOperands.empty()) {
|
|
if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveExpectTxOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTxcount();
|
|
if (getPredicate()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "predicate";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getPredicate())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveExpectTxOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierArriveNoCompleteOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierArriveNoCompleteOpGenericAdaptorBase::MBarrierArriveNoCompleteOpGenericAdaptorBase(MBarrierArriveNoCompleteOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierArriveNoCompleteOpAdaptor::MBarrierArriveNoCompleteOpAdaptor(MBarrierArriveNoCompleteOp op) : MBarrierArriveNoCompleteOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierArriveNoCompleteOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierArriveNoCompleteOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierArriveNoCompleteOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addOperands(count);
|
|
odsState.addTypes(token);
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(token)>(token), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(token)>(token), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addOperands(count);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addOperands(count);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId, ::mlir::Value count) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(count)>(count));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveNoCompleteOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveNoCompleteOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveNoCompleteOp MBarrierArriveNoCompleteOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveNoCompleteOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::MBarrierTokenType>();
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierArriveNoCompleteOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand countRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> countOperands(&countRawOperand, 1); ::llvm::SMLoc countOperandsLoc;
|
|
(void)countOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
::mlir::Type tokenRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
countOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(countRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierTokenType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tokenRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(tokenTypes);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(countOperands, odsBuildableType0, countOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveNoCompleteOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getCount();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getToken().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveNoCompleteOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierArriveOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierArriveOpGenericAdaptorBase::MBarrierArriveOpGenericAdaptorBase(MBarrierArriveOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierArriveOpAdaptor::MBarrierArriveOpAdaptor(MBarrierArriveOp op) : MBarrierArriveOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierArriveOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierArriveOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierArriveOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierArriveOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierArriveOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addTypes(token);
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(token)>(token), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type token, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(token)>(token), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void MBarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierArriveOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierArriveOp MBarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierArriveOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
::mlir::Type odsInferredType0 = odsBuilder.getType<::mlir::nvgpu::MBarrierTokenType>();
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierArriveOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
::mlir::Type tokenRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierTokenType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tokenRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(tokenTypes);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierArriveOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getToken().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierArriveOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierCreateOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierCreateOpGenericAdaptorBase::MBarrierCreateOpGenericAdaptorBase(MBarrierCreateOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierCreateOpAdaptor::MBarrierCreateOpAdaptor(MBarrierCreateOp op) : MBarrierCreateOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierCreateOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierCreateOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierCreateOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierCreateOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierCreateOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierCreateOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierCreateOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierCreateOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierCreateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type barriers) {
|
|
odsState.addTypes(barriers);
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type barriers) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierCreateOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type barriers) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers));
|
|
}
|
|
|
|
void MBarrierCreateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) {
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierCreateOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes));
|
|
}
|
|
|
|
void MBarrierCreateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierCreateOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierCreateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierCreateOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierCreateOp MBarrierCreateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierCreateOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierCreateOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierCreateOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
result.addTypes(barriersTypes);
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierCreateOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierCreateOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierGetOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierGetOpGenericAdaptorBase::MBarrierGetOpGenericAdaptorBase(MBarrierGetOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierGetOpAdaptor::MBarrierGetOpAdaptor(MBarrierGetOp op) : MBarrierGetOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierGetOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierGetOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierGetOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierGetOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierGetOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierGetOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierGetOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierGetOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierGetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addTypes(mbarrierPointer);
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(mbarrierPointer)>(mbarrierPointer), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierGetOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type mbarrierPointer, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(mbarrierPointer)>(mbarrierPointer), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierGetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(mbarId);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierGetOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierGetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierGetOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierGetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierGetOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierGetOp MBarrierGetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierGetOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps10(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierGetOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierGetOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
::mlir::Type mbarrierPointerRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> mbarrierPointerTypes(&mbarrierPointerRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::Type type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
mbarrierPointerRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(mbarrierPointerTypes);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierGetOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMbarrierPointer().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierGetOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierInitOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierInitOpGenericAdaptorBase::MBarrierInitOpGenericAdaptorBase(MBarrierInitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> MBarrierInitOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 3) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierInitOpAdaptor::MBarrierInitOpAdaptor(MBarrierInitOp op) : MBarrierInitOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierInitOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> MBarrierInitOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 3) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange MBarrierInitOp::getPredicateMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierInitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierInitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierInitOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierInitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierInitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierInitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierInitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierInitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(count);
|
|
odsState.addOperands(mbarId);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(count)>(count), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierInitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(count)>(count), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void MBarrierInitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(count);
|
|
odsState.addOperands(mbarId);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(count)>(count), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierInitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value count, ::mlir::Value mbarId, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(count)>(count), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void MBarrierInitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierInitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierInitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierInitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierInitOp MBarrierInitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierInitOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
if (valueGroup3.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup3.size();
|
|
}
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierInitOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierInitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand countRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> countOperands(&countRawOperand, 1); ::llvm::SMLoc countOperandsLoc;
|
|
(void)countOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands;
|
|
::llvm::SMLoc predicateOperandsLoc;
|
|
(void)predicateOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
countOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(countRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("predicate"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
predicateOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
predicateOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(countOperands, odsBuildableType0, countOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!predicateOperands.empty()) {
|
|
if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierInitOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getCount();
|
|
if (getPredicate()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "predicate";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getPredicate())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierInitOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierTestWaitOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierTestWaitOpGenericAdaptorBase::MBarrierTestWaitOpGenericAdaptorBase(MBarrierTestWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierTestWaitOpAdaptor::MBarrierTestWaitOpAdaptor(MBarrierTestWaitOp op) : MBarrierTestWaitOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierTestWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierTestWaitOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierTestWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierTestWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierTestWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(token);
|
|
odsState.addOperands(mbarId);
|
|
odsState.addTypes(waitComplete);
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(waitComplete)>(waitComplete), std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type waitComplete, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(waitComplete)>(waitComplete), std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(token);
|
|
odsState.addOperands(mbarId);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(token);
|
|
odsState.addOperands(mbarId);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value token, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(token)>(token), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void MBarrierTestWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(MBarrierTestWaitOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTestWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTestWaitOp MBarrierTestWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTestWaitOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
::mlir::Type odsInferredType0 = odsBuilder.getIntegerType(1);
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierTestWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand tokenRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tokenOperands(&tokenRawOperand, 1); ::llvm::SMLoc tokenOperandsLoc;
|
|
(void)tokenOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
::mlir::Type tokenRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tokenTypes(&tokenRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
tokenOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tokenRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierTokenType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tokenRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1);
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType();
|
|
result.addTypes(odsBuildableType0);
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(tokenOperands, tokenTypes, tokenOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType1, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierTestWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getToken();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getToken().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierTokenType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierTestWaitOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MBarrierTryWaitParityOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MBarrierTryWaitParityOpGenericAdaptorBase::MBarrierTryWaitParityOpGenericAdaptorBase(MBarrierTryWaitParityOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
MBarrierTryWaitParityOpAdaptor::MBarrierTryWaitParityOpAdaptor(MBarrierTryWaitParityOp op) : MBarrierTryWaitParityOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MBarrierTryWaitParityOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTryWaitParityOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MBarrierTryWaitParityOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MBarrierTryWaitParityOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MBarrierTryWaitParityOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(phaseParity);
|
|
odsState.addOperands(ticks);
|
|
odsState.addOperands(mbarId);
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(barriers)>(barriers), std::forward<decltype(phaseParity)>(phaseParity), std::forward<decltype(ticks)>(ticks), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTryWaitParityOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(barriers)>(barriers), std::forward<decltype(phaseParity)>(phaseParity), std::forward<decltype(ticks)>(ticks), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(phaseParity);
|
|
odsState.addOperands(ticks);
|
|
odsState.addOperands(mbarId);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(phaseParity)>(phaseParity), std::forward<decltype(ticks)>(ticks), std::forward<decltype(mbarId)>(mbarId));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTryWaitParityOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value barriers, ::mlir::Value phaseParity, ::mlir::Value ticks, ::mlir::Value mbarId) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(barriers)>(barriers), std::forward<decltype(phaseParity)>(phaseParity), std::forward<decltype(ticks)>(ticks), std::forward<decltype(mbarId)>(mbarId));
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTryWaitParityOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MBarrierTryWaitParityOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MBarrierTryWaitParityOp MBarrierTryWaitParityOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MBarrierTryWaitParityOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult MBarrierTryWaitParityOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand phaseParityRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> phaseParityOperands(&phaseParityRawOperand, 1); ::llvm::SMLoc phaseParityOperandsLoc;
|
|
(void)phaseParityOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand ticksRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> ticksOperands(&ticksRawOperand, 1); ::llvm::SMLoc ticksOperandsLoc;
|
|
(void)ticksOperandsLoc;
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
phaseParityOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(phaseParityRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
ticksOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(ticksRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1);
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType();
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(phaseParityOperands, odsBuildableType0, phaseParityOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(ticksOperands, odsBuildableType1, ticksOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType1, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MBarrierTryWaitParityOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getPhaseParity();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTicks();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MBarrierTryWaitParityOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MmaSparseSyncOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MmaSparseSyncOpGenericAdaptorBase::MmaSparseSyncOpGenericAdaptorBase(MmaSparseSyncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::ArrayAttr MmaSparseSyncOpGenericAdaptorBase::getMmaShape() {
|
|
auto attr = getMmaShapeAttr();
|
|
return attr;
|
|
}
|
|
|
|
::mlir::IntegerAttr MmaSparseSyncOpGenericAdaptorBase::getSparsitySelectorAttr() {
|
|
auto attr = ::llvm::dyn_cast_or_null<::mlir::IntegerAttr>(getProperties().sparsitySelector);
|
|
return attr;
|
|
}
|
|
|
|
uint32_t MmaSparseSyncOpGenericAdaptorBase::getSparsitySelector() {
|
|
auto attr = getSparsitySelectorAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
::std::optional<bool> MmaSparseSyncOpGenericAdaptorBase::getTf32Enabled() {
|
|
auto attr = getTf32EnabledAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
MmaSparseSyncOpAdaptor::MmaSparseSyncOpAdaptor(MmaSparseSyncOp op) : MmaSparseSyncOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape;
|
|
if (!tblgen_mmaShape) return emitError(loc, "'nvgpu.mma.sp.sync' op requires attribute 'mmaShape'");
|
|
auto tblgen_sparsitySelector = getProperties().sparsitySelector; (void)tblgen_sparsitySelector;
|
|
auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled;
|
|
|
|
if (tblgen_mmaShape && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_mmaShape))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_mmaShape), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); }))))
|
|
return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'mmaShape' failed to satisfy constraint: 64-bit integer array attribute");
|
|
|
|
if (tblgen_sparsitySelector && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_sparsitySelector))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_sparsitySelector).getType().isSignlessInteger(32)))))
|
|
return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'sparsitySelector' failed to satisfy constraint: 32-bit signless integer attribute");
|
|
|
|
if (tblgen_tf32Enabled && !((::llvm::isa<::mlir::UnitAttr>(tblgen_tf32Enabled))))
|
|
return emitError(loc, "'nvgpu.mma.sp.sync' op attribute 'tf32Enabled' failed to satisfy constraint: unit attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.mmaShape;
|
|
auto attr = dict.get("mmaShape");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `mmaShape` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.sparsitySelector;
|
|
auto attr = dict.get("sparsitySelector");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `sparsitySelector` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.tf32Enabled;
|
|
auto attr = dict.get("tf32Enabled");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `tf32Enabled` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MmaSparseSyncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.mmaShape;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("mmaShape",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.sparsitySelector;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("sparsitySelector",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.tf32Enabled;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("tf32Enabled",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MmaSparseSyncOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.mmaShape.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.sparsitySelector.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.tf32Enabled.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MmaSparseSyncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "mmaShape")
|
|
return prop.mmaShape;
|
|
|
|
if (name == "sparsitySelector")
|
|
return prop.sparsitySelector;
|
|
|
|
if (name == "tf32Enabled")
|
|
return prop.tf32Enabled;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MmaSparseSyncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "mmaShape") {
|
|
prop.mmaShape = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.mmaShape)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "sparsitySelector") {
|
|
prop.sparsitySelector = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.sparsitySelector)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "tf32Enabled") {
|
|
prop.tf32Enabled = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.tf32Enabled)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void MmaSparseSyncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.mmaShape) attrs.append("mmaShape", prop.mmaShape);
|
|
|
|
if (prop.sparsitySelector) attrs.append("sparsitySelector", prop.sparsitySelector);
|
|
|
|
if (prop.tf32Enabled) attrs.append("tf32Enabled", prop.tf32Enabled);
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getMmaShapeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(attr, "mmaShape", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getSparsitySelectorAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(attr, "sparsitySelector", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTf32EnabledAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "tf32Enabled", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.mmaShape)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.sparsitySelector)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.tf32Enabled)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MmaSparseSyncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.mmaShape);
|
|
|
|
writer.writeOptionalAttribute(prop.sparsitySelector);
|
|
|
|
writer.writeOptionalAttribute(prop.tf32Enabled);
|
|
}
|
|
|
|
::mlir::ArrayAttr MmaSparseSyncOp::getMmaShape() {
|
|
auto attr = getMmaShapeAttr();
|
|
return attr;
|
|
}
|
|
|
|
uint32_t MmaSparseSyncOp::getSparsitySelector() {
|
|
auto attr = getSparsitySelectorAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
::std::optional<bool> MmaSparseSyncOp::getTf32Enabled() {
|
|
auto attr = getTf32EnabledAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
void MmaSparseSyncOp::setSparsitySelector(uint32_t attrValue) {
|
|
getProperties().sparsitySelector = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue);
|
|
}
|
|
|
|
void MmaSparseSyncOp::setTf32Enabled(bool attrValue) {
|
|
auto &odsProp = getProperties().tf32Enabled;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, Value sparseMetadata, ArrayRef<int64_t> mmaShape) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, Value sparseMetadata, ArrayRef<int64_t> mmaShape) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.addOperands(sparseMetadata);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
if (sparsitySelector) {
|
|
odsState.getOrAddProperties<Properties>().sparsitySelector = sparsitySelector;
|
|
}
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.addOperands(sparseMetadata);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
if (sparsitySelector) {
|
|
odsState.getOrAddProperties<Properties>().sparsitySelector = sparsitySelector;
|
|
}
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, ::mlir::IntegerAttr sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.addOperands(sparseMetadata);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
odsState.getOrAddProperties<Properties>().sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), sparsitySelector);
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.addOperands(sparseMetadata);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
odsState.getOrAddProperties<Properties>().sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), sparsitySelector);
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::Value sparseMetadata, ::mlir::ArrayAttr mmaShape, uint32_t sparsitySelector, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(sparseMetadata)>(sparseMetadata), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(sparsitySelector)>(sparsitySelector), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<MmaSparseSyncOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MmaSparseSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSparseSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSparseSyncOp MmaSparseSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void MmaSparseSyncOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) {
|
|
::mlir::Builder odsBuilder(opName.getContext());
|
|
if (!properties.sparsitySelector)
|
|
properties.sparsitySelector = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 0);
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOp::verifyInvariantsImpl() {
|
|
auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape;
|
|
if (!tblgen_mmaShape) return emitOpError("requires attribute 'mmaShape'");
|
|
auto tblgen_sparsitySelector = getProperties().sparsitySelector; (void)tblgen_sparsitySelector;
|
|
auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(*this, tblgen_mmaShape, "mmaShape")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps3(*this, tblgen_sparsitySelector, "sparsitySelector")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_tf32Enabled, "tf32Enabled")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps11(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!((((*this->getOperation()).getNumOperands() > 1)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(1).getType()))) && ((::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(0)) == ::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(1))))))
|
|
return emitOpError("failed to verify that matrixA and matrixB have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSparseSyncOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult MmaSparseSyncOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixARawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixAOperands(&matrixARawOperand, 1); ::llvm::SMLoc matrixAOperandsLoc;
|
|
(void)matrixAOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixBRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixBOperands(&matrixBRawOperand, 1); ::llvm::SMLoc matrixBOperandsLoc;
|
|
(void)matrixBOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc;
|
|
(void)matrixCOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand sparseMetadataRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sparseMetadataOperands(&sparseMetadataRawOperand, 1); ::llvm::SMLoc sparseMetadataOperandsLoc;
|
|
(void)sparseMetadataOperandsLoc;
|
|
::mlir::Type matrixARawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixATypes(&matrixARawType, 1);
|
|
::mlir::Type matrixBRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixBTypes(&matrixBRawType, 1);
|
|
::mlir::Type matrixCRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1);
|
|
::mlir::Type resRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1);
|
|
if (parser.parseLParen())
|
|
return ::mlir::failure();
|
|
|
|
matrixAOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixARawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
matrixBOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixBRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
matrixCOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixCRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRParen())
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("metadata"))
|
|
return ::mlir::failure();
|
|
if (parser.parseLParen())
|
|
return ::mlir::failure();
|
|
|
|
sparseMetadataOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sparseMetadataRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRParen())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
if (parser.parseLParen())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixARawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixBRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixCRawType = type;
|
|
}
|
|
if (parser.parseRParen())
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
resRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = ::mlir::VectorType::get({2},parser.getBuilder().getI16Type());
|
|
result.addTypes(resTypes);
|
|
if (parser.resolveOperands(matrixAOperands, matrixATypes, matrixAOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(matrixBOperands, matrixBTypes, matrixBOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(sparseMetadataOperands, odsBuildableType0, sparseMetadataOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MmaSparseSyncOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << "(";
|
|
_odsPrinter << getMatrixA();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixB();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixC();
|
|
_odsPrinter << ")";
|
|
_odsPrinter << ' ' << "metadata";
|
|
_odsPrinter << "(";
|
|
_odsPrinter << getSparseMetadata();
|
|
_odsPrinter << ")";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
{
|
|
::mlir::Builder odsBuilder(getContext());
|
|
::mlir::Attribute attr = getSparsitySelectorAttr();
|
|
if(attr && (attr == odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 0)))
|
|
elidedAttrs.push_back("sparsitySelector");
|
|
}
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ' << "(";
|
|
{
|
|
auto type = getMatrixA().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixB().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixC().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ")";
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getRes().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void MmaSparseSyncOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MmaSparseSyncOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::MmaSparseSyncOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::MmaSyncOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
MmaSyncOpGenericAdaptorBase::MmaSyncOpGenericAdaptorBase(MmaSyncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::ArrayAttr MmaSyncOpGenericAdaptorBase::getMmaShape() {
|
|
auto attr = getMmaShapeAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> MmaSyncOpGenericAdaptorBase::getTf32Enabled() {
|
|
auto attr = getTf32EnabledAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
MmaSyncOpAdaptor::MmaSyncOpAdaptor(MmaSyncOp op) : MmaSyncOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult MmaSyncOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape;
|
|
if (!tblgen_mmaShape) return emitError(loc, "'nvgpu.mma.sync' op requires attribute 'mmaShape'");
|
|
auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled;
|
|
|
|
if (tblgen_mmaShape && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_mmaShape))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_mmaShape), [&](::mlir::Attribute attr) { return attr && (((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))); }))))
|
|
return emitError(loc, "'nvgpu.mma.sync' op attribute 'mmaShape' failed to satisfy constraint: 64-bit integer array attribute");
|
|
|
|
if (tblgen_tf32Enabled && !((::llvm::isa<::mlir::UnitAttr>(tblgen_tf32Enabled))))
|
|
return emitError(loc, "'nvgpu.mma.sync' op attribute 'tf32Enabled' failed to satisfy constraint: unit attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSyncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.mmaShape;
|
|
auto attr = dict.get("mmaShape");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `mmaShape` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.tf32Enabled;
|
|
auto attr = dict.get("tf32Enabled");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `tf32Enabled` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute MmaSyncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.mmaShape;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("mmaShape",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.tf32Enabled;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("tf32Enabled",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code MmaSyncOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.mmaShape.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.tf32Enabled.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> MmaSyncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "mmaShape")
|
|
return prop.mmaShape;
|
|
|
|
if (name == "tf32Enabled")
|
|
return prop.tf32Enabled;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void MmaSyncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "mmaShape") {
|
|
prop.mmaShape = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.mmaShape)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "tf32Enabled") {
|
|
prop.tf32Enabled = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.tf32Enabled)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void MmaSyncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.mmaShape) attrs.append("mmaShape", prop.mmaShape);
|
|
|
|
if (prop.tf32Enabled) attrs.append("tf32Enabled", prop.tf32Enabled);
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSyncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getMmaShapeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(attr, "mmaShape", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTf32EnabledAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "tf32Enabled", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSyncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.mmaShape)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.tf32Enabled)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MmaSyncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.mmaShape);
|
|
|
|
writer.writeOptionalAttribute(prop.tf32Enabled);
|
|
}
|
|
|
|
::mlir::ArrayAttr MmaSyncOp::getMmaShape() {
|
|
auto attr = getMmaShapeAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> MmaSyncOp::getTf32Enabled() {
|
|
auto attr = getTf32EnabledAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
void MmaSyncOp::setTf32Enabled(bool attrValue) {
|
|
auto &odsProp = getProperties().tf32Enabled;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, ArrayAttr mmaShape) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, ArrayAttr mmaShape) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape));
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value matrixA, Value matrixB, Value matrixC, ArrayRef<int64_t> mmaShape, bool tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, Value matrixA, Value matrixB, Value matrixC, ArrayRef<int64_t> mmaShape, bool tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
odsState.addOperands(matrixA);
|
|
odsState.addOperands(matrixB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.getOrAddProperties<Properties>().mmaShape = mmaShape;
|
|
if (tf32Enabled) {
|
|
odsState.getOrAddProperties<Properties>().tf32Enabled = tf32Enabled;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixA, ::mlir::Value matrixB, ::mlir::Value matrixC, ::mlir::ArrayAttr mmaShape, /*optional*/::mlir::UnitAttr tf32Enabled) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixA)>(matrixA), std::forward<decltype(matrixB)>(matrixB), std::forward<decltype(matrixC)>(matrixC), std::forward<decltype(mmaShape)>(mmaShape), std::forward<decltype(tf32Enabled)>(tf32Enabled));
|
|
}
|
|
|
|
void MmaSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<MmaSyncOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void MmaSyncOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<MmaSyncOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
MmaSyncOp MmaSyncOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSyncOp::verifyInvariantsImpl() {
|
|
auto tblgen_mmaShape = getProperties().mmaShape; (void)tblgen_mmaShape;
|
|
if (!tblgen_mmaShape) return emitOpError("requires attribute 'mmaShape'");
|
|
auto tblgen_tf32Enabled = getProperties().tf32Enabled; (void)tblgen_tf32Enabled;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps5(*this, tblgen_mmaShape, "mmaShape")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_tf32Enabled, "tf32Enabled")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps6(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!((((*this->getOperation()).getNumOperands() > 1)) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(0).getType()))) && ((::llvm::isa<::mlir::ShapedType>((*this->getOperation()).getOperand(1).getType()))) && ((::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(0)) == ::mlir::getElementTypeOrSelf((*this->getOperation()).getOperand(1))))))
|
|
return emitOpError("failed to verify that matrixA and matrixB have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult MmaSyncOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult MmaSyncOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixARawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixAOperands(&matrixARawOperand, 1); ::llvm::SMLoc matrixAOperandsLoc;
|
|
(void)matrixAOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixBRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixBOperands(&matrixBRawOperand, 1); ::llvm::SMLoc matrixBOperandsLoc;
|
|
(void)matrixBOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc;
|
|
(void)matrixCOperandsLoc;
|
|
::mlir::Type matrixARawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixATypes(&matrixARawType, 1);
|
|
::mlir::Type matrixBRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixBTypes(&matrixBRawType, 1);
|
|
::mlir::Type matrixCRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1);
|
|
::mlir::Type resRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1);
|
|
if (parser.parseLParen())
|
|
return ::mlir::failure();
|
|
|
|
matrixAOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixARawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
matrixBOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixBRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
matrixCOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixCRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRParen())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
if (parser.parseLParen())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixARawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixBRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixCRawType = type;
|
|
}
|
|
if (parser.parseRParen())
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
resRawType = type;
|
|
}
|
|
result.addTypes(resTypes);
|
|
if (parser.resolveOperands(matrixAOperands, matrixATypes, matrixAOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(matrixBOperands, matrixBTypes, matrixBOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void MmaSyncOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << "(";
|
|
_odsPrinter << getMatrixA();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixB();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixC();
|
|
_odsPrinter << ")";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ' << "(";
|
|
{
|
|
auto type = getMatrixA().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixB().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixC().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ")";
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getRes().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void MmaSyncOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::MmaSyncOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::MmaSyncOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::RcpOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
RcpOpGenericAdaptorBase::RcpOpGenericAdaptorBase(RcpOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::NVVM::FPRoundingModeAttr RcpOpGenericAdaptorBase::getRoundingAttr() {
|
|
auto attr = ::llvm::dyn_cast_or_null<::mlir::NVVM::FPRoundingModeAttr>(getProperties().rounding);
|
|
return attr;
|
|
}
|
|
|
|
::mlir::NVVM::FPRoundingMode RcpOpGenericAdaptorBase::getRounding() {
|
|
auto attr = getRoundingAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::mlir::BoolAttr RcpOpGenericAdaptorBase::getApproxAttr() {
|
|
auto attr = ::llvm::dyn_cast_or_null<::mlir::BoolAttr>(getProperties().approx);
|
|
return attr;
|
|
}
|
|
|
|
bool RcpOpGenericAdaptorBase::getApprox() {
|
|
auto attr = getApproxAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::mlir::BoolAttr RcpOpGenericAdaptorBase::getFtzAttr() {
|
|
auto attr = ::llvm::dyn_cast_or_null<::mlir::BoolAttr>(getProperties().ftz);
|
|
return attr;
|
|
}
|
|
|
|
bool RcpOpGenericAdaptorBase::getFtz() {
|
|
auto attr = getFtzAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
RcpOpAdaptor::RcpOpAdaptor(RcpOp op) : RcpOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult RcpOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_approx = getProperties().approx; (void)tblgen_approx;
|
|
auto tblgen_ftz = getProperties().ftz; (void)tblgen_ftz;
|
|
auto tblgen_rounding = getProperties().rounding; (void)tblgen_rounding;
|
|
|
|
if (tblgen_rounding && !((::llvm::isa<::mlir::NVVM::FPRoundingModeAttr>(tblgen_rounding))))
|
|
return emitError(loc, "'nvgpu.rcp' op attribute 'rounding' failed to satisfy constraint: NVVM FPRoundingMode kind");
|
|
|
|
if (tblgen_approx && !((::llvm::isa<::mlir::BoolAttr>(tblgen_approx))))
|
|
return emitError(loc, "'nvgpu.rcp' op attribute 'approx' failed to satisfy constraint: bool attribute");
|
|
|
|
if (tblgen_ftz && !((::llvm::isa<::mlir::BoolAttr>(tblgen_ftz))))
|
|
return emitError(loc, "'nvgpu.rcp' op attribute 'ftz' failed to satisfy constraint: bool attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.approx;
|
|
auto attr = dict.get("approx");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `approx` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.ftz;
|
|
auto attr = dict.get("ftz");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `ftz` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.rounding;
|
|
auto attr = dict.get("rounding");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `rounding` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute RcpOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.approx;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("approx",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.ftz;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("ftz",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.rounding;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("rounding",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code RcpOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.approx.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.ftz.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.rounding.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> RcpOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "approx")
|
|
return prop.approx;
|
|
|
|
if (name == "ftz")
|
|
return prop.ftz;
|
|
|
|
if (name == "rounding")
|
|
return prop.rounding;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void RcpOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "approx") {
|
|
prop.approx = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.approx)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "ftz") {
|
|
prop.ftz = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.ftz)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "rounding") {
|
|
prop.rounding = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.rounding)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void RcpOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.approx) attrs.append("approx", prop.approx);
|
|
|
|
if (prop.ftz) attrs.append("ftz", prop.ftz);
|
|
|
|
if (prop.rounding) attrs.append("rounding", prop.rounding);
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getApproxAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "approx", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getFtzAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(attr, "ftz", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getRoundingAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps6(attr, "rounding", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.approx)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.ftz)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.rounding)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void RcpOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.approx);
|
|
|
|
writer.writeOptionalAttribute(prop.ftz);
|
|
|
|
writer.writeOptionalAttribute(prop.rounding);
|
|
}
|
|
|
|
::mlir::NVVM::FPRoundingMode RcpOp::getRounding() {
|
|
auto attr = getRoundingAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
bool RcpOp::getApprox() {
|
|
auto attr = getApproxAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
bool RcpOp::getFtz() {
|
|
auto attr = getFtzAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
void RcpOp::setRounding(::mlir::NVVM::FPRoundingMode attrValue) {
|
|
getProperties().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue);
|
|
}
|
|
|
|
void RcpOp::setApprox(bool attrValue) {
|
|
getProperties().approx = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue);
|
|
}
|
|
|
|
void RcpOp::setFtz(bool attrValue) {
|
|
getProperties().ftz = ::mlir::Builder((*this)->getContext()).getBoolAttr(attrValue);
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
odsState.addOperands(in);
|
|
if (rounding) {
|
|
odsState.getOrAddProperties<Properties>().rounding = rounding;
|
|
}
|
|
if (approx) {
|
|
odsState.getOrAddProperties<Properties>().approx = approx;
|
|
}
|
|
if (ftz) {
|
|
odsState.getOrAddProperties<Properties>().ftz = ftz;
|
|
}
|
|
odsState.addTypes(out);
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(out)>(out), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(out)>(out), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
odsState.addOperands(in);
|
|
if (rounding) {
|
|
odsState.getOrAddProperties<Properties>().rounding = rounding;
|
|
}
|
|
if (approx) {
|
|
odsState.getOrAddProperties<Properties>().approx = approx;
|
|
}
|
|
if (ftz) {
|
|
odsState.getOrAddProperties<Properties>().ftz = ftz;
|
|
}
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
odsState.addOperands(in);
|
|
if (rounding) {
|
|
odsState.getOrAddProperties<Properties>().rounding = rounding;
|
|
}
|
|
if (approx) {
|
|
odsState.getOrAddProperties<Properties>().approx = approx;
|
|
}
|
|
if (ftz) {
|
|
odsState.getOrAddProperties<Properties>().ftz = ftz;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingModeAttr rounding, ::mlir::BoolAttr approx, ::mlir::BoolAttr ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
odsState.addOperands(in);
|
|
odsState.getOrAddProperties<Properties>().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding);
|
|
odsState.getOrAddProperties<Properties>().approx = odsBuilder.getBoolAttr(approx);
|
|
odsState.getOrAddProperties<Properties>().ftz = odsBuilder.getBoolAttr(ftz);
|
|
odsState.addTypes(out);
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(out)>(out), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type out, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(out)>(out), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
odsState.addOperands(in);
|
|
odsState.getOrAddProperties<Properties>().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding);
|
|
odsState.getOrAddProperties<Properties>().approx = odsBuilder.getBoolAttr(approx);
|
|
odsState.getOrAddProperties<Properties>().ftz = odsBuilder.getBoolAttr(ftz);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, odsState.operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes)))
|
|
odsState.addTypes(inferredReturnTypes);
|
|
else
|
|
::mlir::detail::reportFatalInferReturnTypesError(odsState);
|
|
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
odsState.addOperands(in);
|
|
odsState.getOrAddProperties<Properties>().rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), rounding);
|
|
odsState.getOrAddProperties<Properties>().approx = odsBuilder.getBoolAttr(approx);
|
|
odsState.getOrAddProperties<Properties>().ftz = odsBuilder.getBoolAttr(ftz);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value in, ::mlir::NVVM::FPRoundingMode rounding, bool approx, bool ftz) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(in)>(in), std::forward<decltype(rounding)>(rounding), std::forward<decltype(approx)>(approx), std::forward<decltype(ftz)>(ftz));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<RcpOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<RcpOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void RcpOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties &>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
|
|
::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
|
|
if (::mlir::succeeded(RcpOp::inferReturnTypes(odsBuilder.getContext(),
|
|
odsState.location, operands,
|
|
odsState.attributes.getDictionary(odsState.getContext()),
|
|
odsState.getRawProperties(),
|
|
odsState.regions, inferredReturnTypes))) {
|
|
assert(inferredReturnTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(inferredReturnTypes);
|
|
} else {
|
|
::llvm::report_fatal_error("Failed to infer result type(s).");
|
|
}
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<RcpOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
RcpOp RcpOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
void RcpOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) {
|
|
::mlir::Builder odsBuilder(opName.getContext());
|
|
if (!properties.rounding)
|
|
properties.rounding = ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), ::mlir::NVVM::FPRoundingMode::NONE);
|
|
if (!properties.approx)
|
|
properties.approx = odsBuilder.getBoolAttr(false);
|
|
if (!properties.ftz)
|
|
properties.ftz = odsBuilder.getBoolAttr(false);
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::verifyInvariantsImpl() {
|
|
auto tblgen_approx = getProperties().approx; (void)tblgen_approx;
|
|
auto tblgen_ftz = getProperties().ftz; (void)tblgen_ftz;
|
|
auto tblgen_rounding = getProperties().rounding; (void)tblgen_rounding;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps6(*this, tblgen_rounding, "rounding")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_approx, "approx")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps4(*this, tblgen_ftz, "ftz")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps12(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps12(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::llvm::LogicalResult RcpOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) {
|
|
inferredReturnTypes.resize(1);
|
|
::mlir::Builder odsBuilder(context);
|
|
if (operands.size() <= 0)
|
|
return ::mlir::failure();
|
|
::mlir::Type odsInferredType0 = operands[0].getType();
|
|
inferredReturnTypes[0] = odsInferredType0;
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::ParseResult RcpOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand inRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> inOperands(&inRawOperand, 1); ::llvm::SMLoc inOperandsLoc;
|
|
(void)inOperandsLoc;
|
|
::mlir::Type outRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> outTypes(&outRawType, 1);
|
|
|
|
inOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(inRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
outRawType = type;
|
|
}
|
|
result.addTypes(outTypes);
|
|
if (parser.resolveOperands(inOperands, outTypes[0], inOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void RcpOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getIn();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
{
|
|
::mlir::Builder odsBuilder(getContext());
|
|
::mlir::Attribute attr = getRoundingAttr();
|
|
if(attr && (attr == ::mlir::NVVM::FPRoundingModeAttr::get(odsBuilder.getContext(), ::mlir::NVVM::FPRoundingMode::NONE)))
|
|
elidedAttrs.push_back("rounding");
|
|
}
|
|
{
|
|
::mlir::Builder odsBuilder(getContext());
|
|
::mlir::Attribute attr = getApproxAttr();
|
|
if(attr && (attr == odsBuilder.getBoolAttr(false)))
|
|
elidedAttrs.push_back("approx");
|
|
}
|
|
{
|
|
::mlir::Builder odsBuilder(getContext());
|
|
::mlir::Attribute attr = getFtzAttr();
|
|
if(attr && (attr == odsBuilder.getBoolAttr(false)))
|
|
elidedAttrs.push_back("ftz");
|
|
}
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getOut().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void RcpOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::RcpOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::RcpOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::TmaAsyncLoadOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TmaAsyncLoadOpGenericAdaptorBase::TmaAsyncLoadOpGenericAdaptorBase(TmaAsyncLoadOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> TmaAsyncLoadOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
TmaAsyncLoadOpAdaptor::TmaAsyncLoadOpAdaptor(TmaAsyncLoadOp op) : TmaAsyncLoadOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> TmaAsyncLoadOp::getODSOperandIndexAndLength(unsigned index) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaAsyncLoadOp::getCoordinatesMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaAsyncLoadOp::getMulticastMaskMutable() {
|
|
auto range = getODSOperandIndexAndLength(5);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(5u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaAsyncLoadOp::getPredicateMutable() {
|
|
auto range = getODSOperandIndexAndLength(6);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(6u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
{
|
|
|
|
auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
|
|
::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {
|
|
return convertFromAttribute(propStorage, propAttr, emitError);
|
|
};
|
|
auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");;
|
|
;
|
|
if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() {
|
|
return emitError() << "for `operandSegmentSizes`: ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TmaAsyncLoadOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.operandSegmentSizes;
|
|
auto attr = [&]() -> ::mlir::Attribute {
|
|
return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage);
|
|
}();
|
|
attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TmaAsyncLoadOp::computePropertiesHash(const Properties &prop) {
|
|
auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code {
|
|
using ::llvm::hash_value;
|
|
return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));;
|
|
};
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
hash_operandSegmentSizes_(prop.operandSegmentSizes));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TmaAsyncLoadOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }();
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TmaAsyncLoadOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") {
|
|
auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value);
|
|
if (!arrAttr) return;
|
|
if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t))
|
|
return;
|
|
llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin());
|
|
return;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncLoadOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }());
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
::mlir::DenseI32ArrayAttr attr;
|
|
if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure();
|
|
if (attr.size() > static_cast<int64_t>(sizeof(propStorage) / sizeof(int32_t))) {
|
|
reader.emitError("size mismatch for operand/result_segment_size");
|
|
return ::mlir::failure();
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>(attr), propStorage.begin());
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
auto readProp = [&]() {
|
|
|
|
if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
return reader.readSparseArray(::llvm::MutableArrayRef(propStorage));
|
|
;
|
|
return ::mlir::success();
|
|
};
|
|
if (::mlir::failed(readProp()))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaAsyncLoadOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage));
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
|
|
if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
writer.writeSparseArray(::llvm::ArrayRef(propStorage));
|
|
;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
odsState.addOperands(coordinates);
|
|
odsState.addOperands(mbarId);
|
|
if (multicastMask)
|
|
odsState.addOperands(multicastMask);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, 1, static_cast<int32_t>(coordinates.size()), 1, (multicastMask ? 1 : 0), (predicate ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(dst)>(dst), std::forward<decltype(barriers)>(barriers), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(multicastMask)>(multicastMask), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncLoadOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(dst)>(dst), std::forward<decltype(barriers)>(barriers), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(multicastMask)>(multicastMask), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaAsyncLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(dst);
|
|
odsState.addOperands(barriers);
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
odsState.addOperands(coordinates);
|
|
odsState.addOperands(mbarId);
|
|
if (multicastMask)
|
|
odsState.addOperands(multicastMask);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, 1, static_cast<int32_t>(coordinates.size()), 1, (multicastMask ? 1 : 0), (predicate ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(barriers)>(barriers), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(multicastMask)>(multicastMask), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncLoadOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value dst, ::mlir::Value barriers, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, ::mlir::Value mbarId, /*optional*/::mlir::Value multicastMask, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(dst)>(dst), std::forward<decltype(barriers)>(barriers), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(mbarId)>(mbarId), std::forward<decltype(multicastMask)>(multicastMask), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaAsyncLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<TmaAsyncLoadOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncLoadOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TmaAsyncLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncLoadOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncLoadOp TmaAsyncLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup4 = getODSOperands(4);
|
|
|
|
for (auto v : valueGroup4) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup5 = getODSOperands(5);
|
|
|
|
if (valueGroup5.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup5.size();
|
|
}
|
|
|
|
for (auto v : valueGroup5) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps14(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup6 = getODSOperands(6);
|
|
|
|
if (valueGroup6.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup6.size();
|
|
}
|
|
|
|
for (auto v : valueGroup6) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncLoadOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult TmaAsyncLoadOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc;
|
|
(void)tensorMapDescriptorOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> coordinatesOperands;
|
|
::llvm::SMLoc coordinatesOperandsLoc;
|
|
(void)coordinatesOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand barriersRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> barriersOperands(&barriersRawOperand, 1); ::llvm::SMLoc barriersOperandsLoc;
|
|
(void)barriersOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mbarIdRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mbarIdOperands(&mbarIdRawOperand, 1); ::llvm::SMLoc mbarIdOperandsLoc;
|
|
(void)mbarIdOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc;
|
|
(void)dstOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> multicastMaskOperands;
|
|
::llvm::SMLoc multicastMaskOperandsLoc;
|
|
(void)multicastMaskOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands;
|
|
::llvm::SMLoc predicateOperandsLoc;
|
|
(void)predicateOperandsLoc;
|
|
::mlir::Type tensorMapDescriptorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1);
|
|
::mlir::Type barriersRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> barriersTypes(&barriersRawType, 1);
|
|
::mlir::Type dstRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1);
|
|
|
|
tensorMapDescriptorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorMapDescriptorRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
coordinatesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(coordinatesOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
barriersOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(barriersRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
mbarIdOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mbarIdRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("to"))
|
|
return ::mlir::failure();
|
|
|
|
dstOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(dstRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalKeyword("multicast_mask"))) {
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
multicastMaskOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
multicastMaskOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("predicate"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
predicateOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
predicateOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapDescriptorRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::MBarrierGroupType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
barriersRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
dstRawType = type;
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, 1, static_cast<int32_t>(coordinatesOperands.size()), 1, static_cast<int32_t>(multicastMaskOperands.size()), static_cast<int32_t>(predicateOperands.size())}), result.getOrAddProperties<TmaAsyncLoadOp::Properties>().operandSegmentSizes.begin());
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(16);
|
|
::mlir::Type odsBuildableType2 = parser.getBuilder().getIntegerType(1);
|
|
if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(barriersOperands, barriersTypes, barriersOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(coordinatesOperands, odsBuildableType0, coordinatesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(mbarIdOperands, odsBuildableType0, mbarIdOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!multicastMaskOperands.empty()) {
|
|
if (parser.resolveOperands(multicastMaskOperands, odsBuildableType1, multicastMaskOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
if (!predicateOperands.empty()) {
|
|
if (parser.resolveOperands(predicateOperands, odsBuildableType2, predicateOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaAsyncLoadOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorMapDescriptor();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getCoordinates();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getBarriers();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getMbarId();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ' ' << "to";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDst();
|
|
if (getMulticastMask()) {
|
|
_odsPrinter << ' ' << "multicast_mask";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getMulticastMask())
|
|
_odsPrinter << value;
|
|
}
|
|
if (getPredicate()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "predicate";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getPredicate())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("operandSegmentSizes");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMapDescriptor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getBarriers().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::MBarrierGroupType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDst().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncLoadOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(0);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaAsyncLoadOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::TmaAsyncLoadOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::TmaAsyncStoreOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TmaAsyncStoreOpGenericAdaptorBase::TmaAsyncStoreOpGenericAdaptorBase(TmaAsyncStoreOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> TmaAsyncStoreOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
TmaAsyncStoreOpAdaptor::TmaAsyncStoreOpAdaptor(TmaAsyncStoreOp op) : TmaAsyncStoreOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> TmaAsyncStoreOp::getODSOperandIndexAndLength(unsigned index) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaAsyncStoreOp::getCoordinatesMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaAsyncStoreOp::getPredicateMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
{
|
|
|
|
auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
|
|
::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {
|
|
return convertFromAttribute(propStorage, propAttr, emitError);
|
|
};
|
|
auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");;
|
|
;
|
|
if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() {
|
|
return emitError() << "for `operandSegmentSizes`: ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TmaAsyncStoreOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.operandSegmentSizes;
|
|
auto attr = [&]() -> ::mlir::Attribute {
|
|
return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage);
|
|
}();
|
|
attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TmaAsyncStoreOp::computePropertiesHash(const Properties &prop) {
|
|
auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code {
|
|
using ::llvm::hash_value;
|
|
return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));;
|
|
};
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
hash_operandSegmentSizes_(prop.operandSegmentSizes));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TmaAsyncStoreOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }();
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TmaAsyncStoreOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") {
|
|
auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value);
|
|
if (!arrAttr) return;
|
|
if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t))
|
|
return;
|
|
llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin());
|
|
return;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncStoreOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }());
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
::mlir::DenseI32ArrayAttr attr;
|
|
if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure();
|
|
if (attr.size() > static_cast<int64_t>(sizeof(propStorage) / sizeof(int32_t))) {
|
|
reader.emitError("size mismatch for operand/result_segment_size");
|
|
return ::mlir::failure();
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>(attr), propStorage.begin());
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
auto readProp = [&]() {
|
|
|
|
if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
return reader.readSparseArray(::llvm::MutableArrayRef(propStorage));
|
|
;
|
|
return ::mlir::success();
|
|
};
|
|
if (::mlir::failed(readProp()))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaAsyncStoreOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage));
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
|
|
if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
writer.writeSparseArray(::llvm::ArrayRef(propStorage));
|
|
;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
odsState.addOperands(coordinates);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, static_cast<int32_t>(coordinates.size()), (predicate ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(src)>(src), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(src)>(src), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaAsyncStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(src);
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
odsState.addOperands(coordinates);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, static_cast<int32_t>(coordinates.size()), (predicate ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(src)>(src), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value tensorMapDescriptor, ::mlir::ValueRange coordinates, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(src)>(src), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(coordinates)>(coordinates), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaAsyncStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<TmaAsyncStoreOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TmaAsyncStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaAsyncStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaAsyncStoreOp TmaAsyncStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
if (valueGroup3.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup3.size();
|
|
}
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaAsyncStoreOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult TmaAsyncStoreOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc;
|
|
(void)srcOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc;
|
|
(void)tensorMapDescriptorOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> coordinatesOperands;
|
|
::llvm::SMLoc coordinatesOperandsLoc;
|
|
(void)coordinatesOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands;
|
|
::llvm::SMLoc predicateOperandsLoc;
|
|
(void)predicateOperandsLoc;
|
|
::mlir::Type srcRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1);
|
|
::mlir::Type tensorMapDescriptorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1);
|
|
|
|
srcOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(srcRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("to"))
|
|
return ::mlir::failure();
|
|
|
|
tensorMapDescriptorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorMapDescriptorRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
coordinatesOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(coordinatesOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("predicate"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
predicateOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
predicateOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
srcRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapDescriptorRawType = type;
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, static_cast<int32_t>(coordinatesOperands.size()), static_cast<int32_t>(predicateOperands.size())}), result.getOrAddProperties<TmaAsyncStoreOp::Properties>().operandSegmentSizes.begin());
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(1);
|
|
if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(coordinatesOperands, odsBuildableType0, coordinatesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!predicateOperands.empty()) {
|
|
if (parser.resolveOperands(predicateOperands, odsBuildableType1, predicateOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaAsyncStoreOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSrc();
|
|
_odsPrinter << ' ' << "to";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorMapDescriptor();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getCoordinates();
|
|
_odsPrinter << "]";
|
|
if (getPredicate()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "predicate";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getPredicate())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("operandSegmentSizes");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSrc().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMapDescriptor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void TmaAsyncStoreOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(0);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(1);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaAsyncStoreOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::TmaAsyncStoreOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::TmaCreateDescriptorOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TmaCreateDescriptorOpGenericAdaptorBase::TmaCreateDescriptorOpGenericAdaptorBase(TmaCreateDescriptorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> TmaCreateDescriptorOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
TmaCreateDescriptorOpAdaptor::TmaCreateDescriptorOpAdaptor(TmaCreateDescriptorOp op) : TmaCreateDescriptorOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TmaCreateDescriptorOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> TmaCreateDescriptorOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaCreateDescriptorOp::getBoxDimensionsMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult TmaCreateDescriptorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TmaCreateDescriptorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TmaCreateDescriptorOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TmaCreateDescriptorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult TmaCreateDescriptorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
odsState.addOperands(tensor);
|
|
odsState.addOperands(boxDimensions);
|
|
odsState.addTypes(tensorMap);
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(tensorMap)>(tensorMap), std::forward<decltype(tensor)>(tensor), std::forward<decltype(boxDimensions)>(boxDimensions));
|
|
auto __res__ = ::llvm::dyn_cast<TmaCreateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type tensorMap, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(tensorMap)>(tensorMap), std::forward<decltype(tensor)>(tensor), std::forward<decltype(boxDimensions)>(boxDimensions));
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
odsState.addOperands(tensor);
|
|
odsState.addOperands(boxDimensions);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensor)>(tensor), std::forward<decltype(boxDimensions)>(boxDimensions));
|
|
auto __res__ = ::llvm::dyn_cast<TmaCreateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::ValueRange boxDimensions) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensor)>(tensor), std::forward<decltype(boxDimensions)>(boxDimensions));
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaCreateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaCreateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaCreateDescriptorOp TmaCreateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TmaCreateDescriptorOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps15(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaCreateDescriptorOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult TmaCreateDescriptorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorOperands(&tensorRawOperand, 1); ::llvm::SMLoc tensorOperandsLoc;
|
|
(void)tensorOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> boxDimensionsOperands;
|
|
::llvm::SMLoc boxDimensionsOperandsLoc;
|
|
(void)boxDimensionsOperandsLoc;
|
|
::mlir::Type tensorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorTypes(&tensorRawType, 1);
|
|
::mlir::Type tensorMapRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapTypes(&tensorMapRawType, 1);
|
|
|
|
tensorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("box"))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
boxDimensionsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperandList(boxDimensionsOperands))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::UnrankedMemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(tensorMapTypes);
|
|
if (parser.resolveOperands(tensorOperands, tensorTypes, tensorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(boxDimensionsOperands, odsBuildableType0, boxDimensionsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaCreateDescriptorOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensor();
|
|
_odsPrinter << ' ' << "box";
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getBoxDimensions();
|
|
_odsPrinter << "]";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::UnrankedMemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMap().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaCreateDescriptorOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::TmaFenceOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TmaFenceOpGenericAdaptorBase::TmaFenceOpGenericAdaptorBase(TmaFenceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
TmaFenceOpAdaptor::TmaFenceOpAdaptor(TmaFenceOp op) : TmaFenceOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TmaFenceOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaFenceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TmaFenceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TmaFenceOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TmaFenceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TmaFenceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void TmaFenceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult TmaFenceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaFenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tensorMapDescriptor) {
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tensorMapDescriptor) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor));
|
|
auto __res__ = ::llvm::dyn_cast<TmaFenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tensorMapDescriptor) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor));
|
|
}
|
|
|
|
void TmaFenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) {
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor));
|
|
auto __res__ = ::llvm::dyn_cast<TmaFenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor));
|
|
}
|
|
|
|
void TmaFenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaFenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TmaFenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaFenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaFenceOp TmaFenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TmaFenceOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaFenceOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult TmaFenceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc;
|
|
(void)tensorMapDescriptorOperandsLoc;
|
|
::mlir::Type tensorMapDescriptorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1);
|
|
|
|
tensorMapDescriptorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorMapDescriptorRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapDescriptorRawType = type;
|
|
}
|
|
if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaFenceOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorMapDescriptor();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMapDescriptor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaFenceOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::TmaPrefetchOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TmaPrefetchOpGenericAdaptorBase::TmaPrefetchOpGenericAdaptorBase(TmaPrefetchOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> TmaPrefetchOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
TmaPrefetchOpAdaptor::TmaPrefetchOpAdaptor(TmaPrefetchOp op) : TmaPrefetchOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TmaPrefetchOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> TmaPrefetchOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange TmaPrefetchOp::getPredicateMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult TmaPrefetchOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TmaPrefetchOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TmaPrefetchOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TmaPrefetchOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TmaPrefetchOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void TmaPrefetchOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult TmaPrefetchOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaPrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaPrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaPrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
odsState.addOperands(tensorMapDescriptor);
|
|
if (predicate)
|
|
odsState.addOperands(predicate);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(predicate)>(predicate));
|
|
auto __res__ = ::llvm::dyn_cast<TmaPrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensorMapDescriptor, /*optional*/::mlir::Value predicate) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensorMapDescriptor)>(tensorMapDescriptor), std::forward<decltype(predicate)>(predicate));
|
|
}
|
|
|
|
void TmaPrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaPrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TmaPrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TmaPrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TmaPrefetchOp TmaPrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TmaPrefetchOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
if (valueGroup1.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup1.size();
|
|
}
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TmaPrefetchOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult TmaPrefetchOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorMapDescriptorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapDescriptorOperands(&tensorMapDescriptorRawOperand, 1); ::llvm::SMLoc tensorMapDescriptorOperandsLoc;
|
|
(void)tensorMapDescriptorOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> predicateOperands;
|
|
::llvm::SMLoc predicateOperandsLoc;
|
|
(void)predicateOperandsLoc;
|
|
::mlir::Type tensorMapDescriptorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapDescriptorTypes(&tensorMapDescriptorRawType, 1);
|
|
|
|
tensorMapDescriptorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorMapDescriptorRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("predicate"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
predicateOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
predicateOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapDescriptorRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1);
|
|
if (parser.resolveOperands(tensorMapDescriptorOperands, tensorMapDescriptorTypes, tensorMapDescriptorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!predicateOperands.empty()) {
|
|
if (parser.resolveOperands(predicateOperands, odsBuildableType0, predicateOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TmaPrefetchOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorMapDescriptor();
|
|
if (getPredicate()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "predicate";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getPredicate())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMapDescriptor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::TmaPrefetchOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::WarpgroupGenerateDescriptorOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
WarpgroupGenerateDescriptorOpGenericAdaptorBase::WarpgroupGenerateDescriptorOpGenericAdaptorBase(WarpgroupGenerateDescriptorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
WarpgroupGenerateDescriptorOpAdaptor::WarpgroupGenerateDescriptorOpAdaptor(WarpgroupGenerateDescriptorOp op) : WarpgroupGenerateDescriptorOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult WarpgroupGenerateDescriptorOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupGenerateDescriptorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute WarpgroupGenerateDescriptorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code WarpgroupGenerateDescriptorOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> WarpgroupGenerateDescriptorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
odsState.addOperands(tensor);
|
|
odsState.addOperands(tensorMap);
|
|
odsState.addTypes(descriptor);
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(descriptor)>(descriptor), std::forward<decltype(tensor)>(tensor), std::forward<decltype(tensorMap)>(tensorMap));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupGenerateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type descriptor, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(descriptor)>(descriptor), std::forward<decltype(tensor)>(tensor), std::forward<decltype(tensorMap)>(tensorMap));
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
odsState.addOperands(tensor);
|
|
odsState.addOperands(tensorMap);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensor)>(tensor), std::forward<decltype(tensorMap)>(tensorMap));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupGenerateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tensor, ::mlir::Value tensorMap) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(tensor)>(tensor), std::forward<decltype(tensorMap)>(tensorMap));
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupGenerateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupGenerateDescriptorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupGenerateDescriptorOp WarpgroupGenerateDescriptorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupGenerateDescriptorOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult WarpgroupGenerateDescriptorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorOperands(&tensorRawOperand, 1); ::llvm::SMLoc tensorOperandsLoc;
|
|
(void)tensorOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorMapRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorMapOperands(&tensorMapRawOperand, 1); ::llvm::SMLoc tensorMapOperandsLoc;
|
|
(void)tensorMapOperandsLoc;
|
|
::mlir::Type tensorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorTypes(&tensorRawType, 1);
|
|
::mlir::Type tensorMapRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorMapTypes(&tensorMapRawType, 1);
|
|
::mlir::Type descriptorRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> descriptorTypes(&descriptorRawType, 1);
|
|
|
|
tensorOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
tensorMapOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorMapRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::TensorMapDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
tensorMapRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupMatrixDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
descriptorRawType = type;
|
|
}
|
|
result.addTypes(descriptorTypes);
|
|
if (parser.resolveOperands(tensorOperands, tensorTypes, tensorOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(tensorMapOperands, tensorMapTypes, tensorMapOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensor();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorMap();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getTensorMap().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::TensorMapDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDescriptor().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void WarpgroupGenerateDescriptorOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(0);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupGenerateDescriptorOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
WarpgroupMmaInitAccumulatorOpGenericAdaptorBase::WarpgroupMmaInitAccumulatorOpGenericAdaptorBase(WarpgroupMmaInitAccumulatorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
WarpgroupMmaInitAccumulatorOpAdaptor::WarpgroupMmaInitAccumulatorOpAdaptor(WarpgroupMmaInitAccumulatorOp op) : WarpgroupMmaInitAccumulatorOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaInitAccumulatorOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute WarpgroupMmaInitAccumulatorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code WarpgroupMmaInitAccumulatorOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> WarpgroupMmaInitAccumulatorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixC) {
|
|
odsState.addTypes(matrixC);
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixC) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixC)>(matrixC));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaInitAccumulatorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixC) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixC)>(matrixC));
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) {
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaInitAccumulatorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes));
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaInitAccumulatorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaInitAccumulatorOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaInitAccumulatorOp WarpgroupMmaInitAccumulatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaInitAccumulatorOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult WarpgroupMmaInitAccumulatorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::Type matrixCRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1);
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupAccumulatorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixCRawType = type;
|
|
}
|
|
result.addTypes(matrixCTypes);
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaInitAccumulatorOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixC().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaInitAccumulatorOp)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::WarpgroupMmaOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
WarpgroupMmaOpGenericAdaptorBase::WarpgroupMmaOpGenericAdaptorBase(WarpgroupMmaOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::IntegerAttr WarpgroupMmaOpGenericAdaptorBase::getWaitGroupAttr() {
|
|
auto attr = ::llvm::dyn_cast_or_null<::mlir::IntegerAttr>(getProperties().waitGroup);
|
|
if (!attr)
|
|
attr = ::mlir::Builder(odsAttrs.getContext()).getIntegerAttr(::mlir::Builder(odsAttrs.getContext()).getIntegerType(64), 1);
|
|
return attr;
|
|
}
|
|
|
|
uint64_t WarpgroupMmaOpGenericAdaptorBase::getWaitGroup() {
|
|
auto attr = getWaitGroupAttr();
|
|
if (!attr)
|
|
return ::mlir::Builder(odsAttrs.getContext()).getIntegerAttr(::mlir::Builder(odsAttrs.getContext()).getIntegerType(64), 1).getValue().getZExtValue();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
::std::optional<bool> WarpgroupMmaOpGenericAdaptorBase::getTransposeA() {
|
|
auto attr = getTransposeAAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<bool> WarpgroupMmaOpGenericAdaptorBase::getTransposeB() {
|
|
auto attr = getTransposeBAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
WarpgroupMmaOpAdaptor::WarpgroupMmaOpAdaptor(WarpgroupMmaOp op) : WarpgroupMmaOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_transposeA = getProperties().transposeA; (void)tblgen_transposeA;
|
|
auto tblgen_transposeB = getProperties().transposeB; (void)tblgen_transposeB;
|
|
auto tblgen_waitGroup = getProperties().waitGroup; (void)tblgen_waitGroup;
|
|
|
|
if (tblgen_waitGroup && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_waitGroup))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_waitGroup).getType().isSignlessInteger(64)))))
|
|
return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'waitGroup' failed to satisfy constraint: 64-bit signless integer attribute");
|
|
|
|
if (tblgen_transposeA && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transposeA))))
|
|
return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'transposeA' failed to satisfy constraint: unit attribute");
|
|
|
|
if (tblgen_transposeB && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transposeB))))
|
|
return emitError(loc, "'nvgpu.warpgroup.mma' op attribute 'transposeB' failed to satisfy constraint: unit attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.transposeA;
|
|
auto attr = dict.get("transposeA");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `transposeA` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.transposeB;
|
|
auto attr = dict.get("transposeB");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `transposeB` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.waitGroup;
|
|
auto attr = dict.get("waitGroup");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `waitGroup` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute WarpgroupMmaOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.transposeA;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("transposeA",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.transposeB;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("transposeB",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.waitGroup;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("waitGroup",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code WarpgroupMmaOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.transposeA.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.transposeB.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.waitGroup.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> WarpgroupMmaOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "transposeA")
|
|
return prop.transposeA;
|
|
|
|
if (name == "transposeB")
|
|
return prop.transposeB;
|
|
|
|
if (name == "waitGroup")
|
|
return prop.waitGroup;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void WarpgroupMmaOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "transposeA") {
|
|
prop.transposeA = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.transposeA)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "transposeB") {
|
|
prop.transposeB = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.transposeB)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "waitGroup") {
|
|
prop.waitGroup = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.waitGroup)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void WarpgroupMmaOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.transposeA) attrs.append("transposeA", prop.transposeA);
|
|
|
|
if (prop.transposeB) attrs.append("transposeB", prop.transposeB);
|
|
|
|
if (prop.waitGroup) attrs.append("waitGroup", prop.waitGroup);
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTransposeAAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "transposeA", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTransposeBAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(attr, "transposeB", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getWaitGroupAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps7(attr, "waitGroup", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.transposeA)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.transposeB)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.waitGroup)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.transposeA);
|
|
|
|
writer.writeOptionalAttribute(prop.transposeB);
|
|
|
|
writer.writeOptionalAttribute(prop.waitGroup);
|
|
}
|
|
|
|
uint64_t WarpgroupMmaOp::getWaitGroup() {
|
|
auto attr = getWaitGroupAttr();
|
|
if (!attr)
|
|
return ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), 1).getValue().getZExtValue();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
::std::optional<bool> WarpgroupMmaOp::getTransposeA() {
|
|
auto attr = getTransposeAAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<bool> WarpgroupMmaOp::getTransposeB() {
|
|
auto attr = getTransposeBAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
void WarpgroupMmaOp::setWaitGroup(::std::optional<uint64_t> attrValue) {
|
|
auto &odsProp = getProperties().waitGroup;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void WarpgroupMmaOp::setTransposeA(bool attrValue) {
|
|
auto &odsProp = getProperties().transposeA;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void WarpgroupMmaOp::setTransposeB(bool attrValue) {
|
|
auto &odsProp = getProperties().transposeB;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
odsState.addOperands(descriptorA);
|
|
odsState.addOperands(descriptorB);
|
|
odsState.addOperands(matrixC);
|
|
if (waitGroup) {
|
|
odsState.getOrAddProperties<Properties>().waitGroup = waitGroup;
|
|
}
|
|
if (transposeA) {
|
|
odsState.getOrAddProperties<Properties>().transposeA = transposeA;
|
|
}
|
|
if (transposeB) {
|
|
odsState.getOrAddProperties<Properties>().transposeB = transposeB;
|
|
}
|
|
odsState.addTypes(matrixD);
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
odsState.addOperands(descriptorA);
|
|
odsState.addOperands(descriptorB);
|
|
odsState.addOperands(matrixC);
|
|
if (waitGroup) {
|
|
odsState.getOrAddProperties<Properties>().waitGroup = waitGroup;
|
|
}
|
|
if (transposeA) {
|
|
odsState.getOrAddProperties<Properties>().transposeA = transposeA;
|
|
}
|
|
if (transposeB) {
|
|
odsState.getOrAddProperties<Properties>().transposeB = transposeB;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/::mlir::IntegerAttr waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
odsState.addOperands(descriptorA);
|
|
odsState.addOperands(descriptorB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.getOrAddProperties<Properties>().waitGroup = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), waitGroup);
|
|
if (transposeA) {
|
|
odsState.getOrAddProperties<Properties>().transposeA = transposeA;
|
|
}
|
|
if (transposeB) {
|
|
odsState.getOrAddProperties<Properties>().transposeB = transposeB;
|
|
}
|
|
odsState.addTypes(matrixD);
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type matrixD, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
odsState.addOperands(descriptorA);
|
|
odsState.addOperands(descriptorB);
|
|
odsState.addOperands(matrixC);
|
|
odsState.getOrAddProperties<Properties>().waitGroup = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), waitGroup);
|
|
if (transposeA) {
|
|
odsState.getOrAddProperties<Properties>().transposeA = transposeA;
|
|
}
|
|
if (transposeB) {
|
|
odsState.getOrAddProperties<Properties>().transposeB = transposeB;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value descriptorA, ::mlir::Value descriptorB, /*optional*/uint64_t waitGroup, /*optional*/::mlir::UnitAttr transposeA, /*optional*/::mlir::UnitAttr transposeB, ::mlir::Value matrixC) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(descriptorA)>(descriptorA), std::forward<decltype(descriptorB)>(descriptorB), std::forward<decltype(waitGroup)>(waitGroup), std::forward<decltype(transposeA)>(transposeA), std::forward<decltype(transposeB)>(transposeB), std::forward<decltype(matrixC)>(matrixC));
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<WarpgroupMmaOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void WarpgroupMmaOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaOp WarpgroupMmaOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOp::verifyInvariantsImpl() {
|
|
auto tblgen_transposeA = getProperties().transposeA; (void)tblgen_transposeA;
|
|
auto tblgen_transposeB = getProperties().transposeB; (void)tblgen_transposeB;
|
|
auto tblgen_waitGroup = getProperties().waitGroup; (void)tblgen_waitGroup;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps7(*this, tblgen_waitGroup, "waitGroup")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_transposeA, "transposeA")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_NVGPUOps2(*this, tblgen_transposeB, "transposeB")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps16(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult WarpgroupMmaOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand descriptorARawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descriptorAOperands(&descriptorARawOperand, 1); ::llvm::SMLoc descriptorAOperandsLoc;
|
|
(void)descriptorAOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand descriptorBRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descriptorBOperands(&descriptorBRawOperand, 1); ::llvm::SMLoc descriptorBOperandsLoc;
|
|
(void)descriptorBOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixCRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixCOperands(&matrixCRawOperand, 1); ::llvm::SMLoc matrixCOperandsLoc;
|
|
(void)matrixCOperandsLoc;
|
|
::mlir::Type descriptorARawType{};
|
|
::llvm::ArrayRef<::mlir::Type> descriptorATypes(&descriptorARawType, 1);
|
|
::mlir::Type descriptorBRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> descriptorBTypes(&descriptorBRawType, 1);
|
|
::mlir::Type matrixCRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixCTypes(&matrixCRawType, 1);
|
|
::mlir::Type matrixDRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixDTypes(&matrixDRawType, 1);
|
|
|
|
descriptorAOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(descriptorARawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
descriptorBOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(descriptorBRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
matrixCOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixCRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupMatrixDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
descriptorARawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupMatrixDescriptorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
descriptorBRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupAccumulatorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixCRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupAccumulatorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixDRawType = type;
|
|
}
|
|
result.addTypes(matrixDTypes);
|
|
if (parser.resolveOperands(descriptorAOperands, descriptorATypes, descriptorAOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(descriptorBOperands, descriptorBTypes, descriptorBOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(matrixCOperands, matrixCTypes, matrixCOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDescriptorA();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDescriptorB();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixC();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
{
|
|
::mlir::Builder odsBuilder(getContext());
|
|
::mlir::Attribute attr = getWaitGroupAttr();
|
|
if(attr && (attr == odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), 1)))
|
|
elidedAttrs.push_back("waitGroup");
|
|
}
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDescriptorA().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDescriptorB().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupMatrixDescriptorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixC().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixD().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::detail::WarpgroupMmaOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::nvgpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::nvgpu::WarpgroupMmaStoreOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
WarpgroupMmaStoreOpGenericAdaptorBase::WarpgroupMmaStoreOpGenericAdaptorBase(WarpgroupMmaStoreOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
WarpgroupMmaStoreOpAdaptor::WarpgroupMmaStoreOpAdaptor(WarpgroupMmaStoreOp op) : WarpgroupMmaStoreOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaStoreOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaStoreOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute WarpgroupMmaStoreOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code WarpgroupMmaStoreOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> WarpgroupMmaStoreOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
odsState.addOperands(matrixD);
|
|
odsState.addOperands(dstMemref);
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(dstMemref)>(dstMemref));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(dstMemref)>(dstMemref));
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
odsState.addOperands(matrixD);
|
|
odsState.addOperands(dstMemref);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(dstMemref)>(dstMemref));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrixD, ::mlir::Value dstMemref) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(matrixD)>(matrixD), std::forward<decltype(dstMemref)>(dstMemref));
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<WarpgroupMmaStoreOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
WarpgroupMmaStoreOp WarpgroupMmaStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps17(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_NVGPUOps1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult WarpgroupMmaStoreOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult WarpgroupMmaStoreOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand matrixDRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixDOperands(&matrixDRawOperand, 1); ::llvm::SMLoc matrixDOperandsLoc;
|
|
(void)matrixDOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand dstMemrefRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstMemrefOperands(&dstMemrefRawOperand, 1); ::llvm::SMLoc dstMemrefOperandsLoc;
|
|
(void)dstMemrefOperandsLoc;
|
|
::mlir::Type matrixDRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> matrixDTypes(&matrixDRawType, 1);
|
|
::mlir::Type dstMemrefRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> dstMemrefTypes(&dstMemrefRawType, 1);
|
|
|
|
matrixDOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(matrixDRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
dstMemrefOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(dstMemrefRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::nvgpu::WarpgroupAccumulatorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
matrixDRawType = type;
|
|
}
|
|
if (parser.parseKeyword("to"))
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::MemRefType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
dstMemrefRawType = type;
|
|
}
|
|
if (parser.resolveOperands(matrixDOperands, matrixDTypes, matrixDOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(dstMemrefOperands, dstMemrefTypes, dstMemrefOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMatrixD();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDstMemref();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMatrixD().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::nvgpu::WarpgroupAccumulatorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "to";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getDstMemref().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void WarpgroupMmaStoreOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
{
|
|
auto valueRange = getODSOperandIndexAndLength(1);
|
|
for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
} // namespace mlir::nvgpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::nvgpu::WarpgroupMmaStoreOp)
|
|
|
|
|
|
#endif // GET_OP_CLASSES
|
|
|