/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ |* *| |* Op Definitions *| |* *| |* Automatically generated file, do not edit! *| |* From: GPUOps.td *| |* *| \*===----------------------------------------------------------------------===*/ #ifdef GET_OP_LIST #undef GET_OP_LIST ::mlir::gpu::AllReduceOp, ::mlir::gpu::AllocOp, ::mlir::gpu::BallotOp, ::mlir::gpu::BarrierOp, ::mlir::gpu::BinaryOp, ::mlir::gpu::BlockDimOp, ::mlir::gpu::BlockIdOp, ::mlir::gpu::ClusterBlockIdOp, ::mlir::gpu::ClusterDimBlocksOp, ::mlir::gpu::ClusterDimOp, ::mlir::gpu::ClusterIdOp, ::mlir::gpu::Create2To4SpMatOp, ::mlir::gpu::CreateBsrOp, ::mlir::gpu::CreateCooAoSOp, ::mlir::gpu::CreateCooOp, ::mlir::gpu::CreateCscOp, ::mlir::gpu::CreateCsrOp, ::mlir::gpu::CreateDnTensorOp, ::mlir::gpu::DeallocOp, ::mlir::gpu::DestroyDnTensorOp, ::mlir::gpu::DestroySpMatOp, ::mlir::gpu::DynamicSharedMemoryOp, ::mlir::gpu::GPUFuncOp, ::mlir::gpu::GPUModuleOp, ::mlir::gpu::GlobalIdOp, ::mlir::gpu::GridDimOp, ::mlir::gpu::HostRegisterOp, ::mlir::gpu::HostUnregisterOp, ::mlir::gpu::InitializeNamedBarrierOp, ::mlir::gpu::LaneIdOp, ::mlir::gpu::LaunchFuncOp, ::mlir::gpu::LaunchOp, ::mlir::gpu::MemcpyOp, ::mlir::gpu::MemsetOp, ::mlir::gpu::NumSubgroupsOp, ::mlir::gpu::PrintfOp, ::mlir::gpu::ReturnOp, ::mlir::gpu::RotateOp, ::mlir::gpu::SDDMMBufferSizeOp, ::mlir::gpu::SDDMMOp, ::mlir::gpu::SetCsrPointersOp, ::mlir::gpu::SetDefaultDeviceOp, ::mlir::gpu::ShuffleOp, ::mlir::gpu::SpGEMMCopyOp, ::mlir::gpu::SpGEMMCreateDescrOp, ::mlir::gpu::SpGEMMDestroyDescrOp, ::mlir::gpu::SpGEMMWorkEstimationOrComputeOp, ::mlir::gpu::SpMMBufferSizeOp, ::mlir::gpu::SpMMOp, ::mlir::gpu::SpMVBufferSizeOp, ::mlir::gpu::SpMVOp, ::mlir::gpu::SpMatGetSizeOp, ::mlir::gpu::SubgroupBroadcastOp, ::mlir::gpu::SubgroupIdOp, ::mlir::gpu::SubgroupMmaComputeOp, ::mlir::gpu::SubgroupMmaConstantMatrixOp, ::mlir::gpu::SubgroupMmaElementwiseOp, ::mlir::gpu::SubgroupMmaExtractThreadLocalOp, ::mlir::gpu::SubgroupMmaInsertThreadLocalOp, ::mlir::gpu::SubgroupMmaLoadMatrixOp, ::mlir::gpu::SubgroupMmaStoreMatrixOp, ::mlir::gpu::SubgroupReduceOp, ::mlir::gpu::SubgroupSizeOp, ::mlir::gpu::TerminatorOp, ::mlir::gpu::ThreadIdOp, ::mlir::gpu::WaitOp, ::mlir::gpu::WarpExecuteOnLane0Op, ::mlir::gpu::YieldOp #endif // GET_OP_LIST #ifdef GET_OP_CLASSES #undef GET_OP_CLASSES //===----------------------------------------------------------------------===// // Local Utility Method Definitions //===----------------------------------------------------------------------===// static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps1( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((type.isSignlessInteger())) || ((::llvm::isa<::mlir::FloatType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be Integer or Float, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps2( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::gpu::AsyncTokenType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of async token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps3( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps4( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps5( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::gpu::AsyncTokenType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be async token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps6( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger(1)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 1-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps7( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger()))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps8( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::gpu::NamedBarrierType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be named barrier type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps9( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps10( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((llvm::isa<::mlir::gpu::SparseSpMatHandleType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be sparse matrix handle type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps11( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((llvm::isa<::mlir::gpu::SparseDnTensorHandleType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be dense tensor handle type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps12( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() == 1))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 1D memref of 8-bit signless integer values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps13( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::UnrankedMemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be unranked.memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps14( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger(32)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 32-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps15( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::IndexType>(type))) || ((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be index or 32-bit signless integer or 64-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps16( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(!((::llvm::isa<::mlir::TokenType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of any non-token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps17( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(!((::llvm::isa<::mlir::TokenType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be any non-token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps18( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::IntegerType>(type))) || ((::llvm::isa<::mlir::IndexType>(type))) || ((::llvm::isa<::mlir::FloatType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of integer or index or floating-point, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps19( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((type.isSignlessInteger())) || ((::llvm::isa<::mlir::FloatType>(type)))) || ((((::llvm::isa<::mlir::FixedVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger())) || ((::llvm::isa<::mlir::FloatType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps20( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((llvm::isa<::mlir::gpu::SparseSpGEMMOpHandleType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be SpGEMM operation handle type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps21( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps22( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::gpu::MMAMatrixType>(type))) && (((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isSignedInteger(8))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isUnsignedInteger(8))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF16())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF32())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF64()))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be gpu.mma_matrix of 8-bit signed integer or 8-bit unsigned integer or 16-bit float or 32-bit float or 64-bit float values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps23( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::gpu::MMAMatrixType>(type))) && (((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isSignlessInteger(32))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF16())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF32())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF64()))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be gpu.mma_matrix of 32-bit signless integer or 16-bit float or 32-bit float or 64-bit float values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps24( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::gpu::MMAMatrixType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be MMAMatrix type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps25( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((type.isSignedInteger(8))) || ((type.isUnsignedInteger(8))) || ((type.isSignlessInteger(32))) || ((type.isF16())) || ((type.isF32())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 8-bit signed integer or 8-bit unsigned integer or 32-bit signless integer or 16-bit float or 32-bit float, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps26( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::gpu::MMAMatrixType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of MMAMatrix type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps27( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(32))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || (((((::llvm::isa<::mlir::VectorType>(elementType))) && ((::llvm::cast<::mlir::VectorType>(elementType).getRank() > 0))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(32))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())); }(::llvm::cast<::mlir::ShapedType>(elementType).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(elementType))) && ((::llvm::cast<::mlir::VectorType>(elementType).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(elementType).getRank() == 1)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be memref of 8-bit signless integer or 32-bit signless integer or 16-bit float or 32-bit float or 64-bit float or vector of 8-bit signless integer or 32-bit signless integer or 16-bit float or 32-bit float or 64-bit float values of ranks 1 values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_GPUOps28( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::gpu::MMAMatrixType>(type))) && (((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isSignedInteger(8))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isUnsignedInteger(8))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isSignlessInteger(32))) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF16())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF32())) || ((::llvm::cast<::mlir::gpu::MMAMatrixType>(type).getElementType().isF64()))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be gpu.mma_matrix of 8-bit signed integer or 8-bit unsigned integer or 32-bit signless integer or 16-bit float or 32-bit float or 64-bit float values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps1( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::AllReduceOperationAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: built-in reduction operations supported by gpu.allreduce."; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps1( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps1(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps2( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::UnitAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: unit attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps2( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps2(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps3( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::gpu::AddressSpaceAttr>(attr))); })))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: GPU Address Space array"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps3( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps3(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps4( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::BarrierScopeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: barrier execution scope"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps4( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps4(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps5( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::StringAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: string attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps5( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps5(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps6( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((true)) && ((attr.hasTrait<::mlir::gpu::OffloadingTranslationAttrTrait>())))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: any attribute with the `OffloadingTranslationAttrTrait` trait."; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps6( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps6(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps7( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::gpu::ObjectAttr>(attr))); }))) && ((::llvm::cast<::mlir::ArrayAttr>(attr).size() >= 1)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: an array of GPU object attributes with at least 1 elements"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps7( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps7(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps8( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::DimensionAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps8( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps8(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps9( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(attr).getType()))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: index attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps9( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps9(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps10( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::Prune2To4SpMatFlagAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: pruning strategy for 2:4 sparse matrix"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps10( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps10(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps11( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::TypeAttr>(attr))) && ((::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>(attr).getValue()))) && ((::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>(attr).getValue()))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: type attribute of function type"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps11( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps11(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps12( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::DictionaryAttr>(attr))); })))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Array of dictionary attributes"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps12( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps12(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps13( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::DenseI32ArrayAttr>(attr))) && (((!attr)) || (((::llvm::cast<::mlir::DenseArrayAttr>(attr).size() == 3)) && ((([](mlir::Attribute a) { auto arr = ::llvm::cast<::mlir::DenseI32ArrayAttr>(a).asArrayRef(); return ::llvm::all_of(arr, [](int32_t v) { return v >= 1; });})(attr))))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: i32 dense array attribute with 3 elements (if present) and all elements >= 1"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps13( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps13(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps14( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))) && ((!::llvm::cast<::mlir::IntegerAttr>(attr).getValue().isNegative())))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 64-bit signless integer attribute whose value is non-negative"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps14( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps14(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps15( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((((::llvm::isa<::mlir::ArrayAttr>(attr))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(attr), [&](::mlir::Attribute attr) { return attr && (((true)) && ((attr.hasPromiseOrImplementsInterface<::mlir::gpu::TargetAttrInterface>()))); }))) && ((::llvm::cast<::mlir::ArrayAttr>(attr).size() >= 1)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: array of GPU target attributes with at least 1 elements"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps15( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps15(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps16( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::SymbolRefAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: symbol reference attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps16( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps16(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps17( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::FlatSymbolRefAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: flat symbol reference attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps17( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps17(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps18( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(32)))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getInt() >= 0)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 32-bit signless integer attribute whose minimum value is 0"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps18( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps18(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps19( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(32)))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getValue().isPowerOf2())))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 32-bit signless integer attribute whose value is a power of two > 0"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps19( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps19(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps20( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps20( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps20(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps21( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::TypeAttr>(attr))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(attr).getValue()))) && ((true)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: any type attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps21( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps21(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps22( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::ShuffleModeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Indexing modes supported by gpu.shuffle."; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps22( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps22(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps23( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: choose whether spgemm_work_estimation_or_compute does work estimation or compute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps23( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps23(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps24( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::BroadcastTypeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: a lane to broadcast from"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps24( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps24(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps25( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::gpu::MMAElementwiseOpAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: elementwise operation to apply to mma matrix"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps25( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps25(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps26( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(32))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 32-bit signless integer attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps26( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps26(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps27( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 64-bit signless integer attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_GPUOps27( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_GPUOps27(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_region_constraint_GPUOps1( ::mlir::Operation *op, ::mlir::Region ®ion, ::llvm::StringRef regionName, unsigned regionIndex) { if (!((true))) { return op->emitOpError("region #") << regionIndex << (regionName.empty() ? " " : " ('" + regionName + "') ") << "failed to verify constraint: any region"; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_region_constraint_GPUOps2( ::mlir::Operation *op, ::mlir::Region ®ion, ::llvm::StringRef regionName, unsigned regionIndex) { if (!((::llvm::hasNItems(region, 1)))) { return op->emitOpError("region #") << regionIndex << (regionName.empty() ? " " : " ('" + regionName + "') ") << "failed to verify constraint: region with 1 blocks"; } return ::mlir::success(); } namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::AllReduceOp definitions //===----------------------------------------------------------------------===// namespace detail { AllReduceOpGenericAdaptorBase::AllReduceOpGenericAdaptorBase(AllReduceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional<::mlir::gpu::AllReduceOperation> AllReduceOpGenericAdaptorBase::getOp() { auto attr = getOpAttr(); return attr ? ::std::optional<::mlir::gpu::AllReduceOperation>(attr.getValue()) : (::std::nullopt); } ::mlir::UnitAttr AllReduceOpGenericAdaptorBase::getUniformAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().uniform); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool AllReduceOpGenericAdaptorBase::getUniform() { auto attr = getUniformAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } } // namespace detail AllReduceOpAdaptor::AllReduceOpAdaptor(AllReduceOp op) : AllReduceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult AllReduceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_op = getProperties().op; (void)tblgen_op; auto tblgen_uniform = getProperties().uniform; (void)tblgen_uniform; if (tblgen_op && !((::llvm::isa<::mlir::gpu::AllReduceOperationAttr>(tblgen_op)))) return emitError(loc, "'gpu.all_reduce' op attribute 'op' failed to satisfy constraint: built-in reduction operations supported by gpu.allreduce."); if (tblgen_uniform && !((::llvm::isa<::mlir::UnitAttr>(tblgen_uniform)))) return emitError(loc, "'gpu.all_reduce' op attribute 'uniform' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult AllReduceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.op; auto attr = dict.get("op"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `op` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.uniform; auto attr = dict.get("uniform"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `uniform` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute AllReduceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.op; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("op", propStorage)); } { const auto &propStorage = prop.uniform; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("uniform", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code AllReduceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.op.getAsOpaquePointer()), llvm::hash_value(prop.uniform.getAsOpaquePointer())); } std::optional AllReduceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "op") return prop.op; if (name == "uniform") return prop.uniform; return std::nullopt; } void AllReduceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "op") { prop.op = ::llvm::dyn_cast_or_null>(value); return; } if (name == "uniform") { prop.uniform = ::llvm::dyn_cast_or_null>(value); return; } } void AllReduceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.op) attrs.append("op", prop.op); if (prop.uniform) attrs.append("uniform", prop.uniform); } ::llvm::LogicalResult AllReduceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getOpAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps1(attr, "op", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUniformAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "uniform", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult AllReduceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.op))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.uniform))) return ::mlir::failure(); return ::mlir::success(); } void AllReduceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.op); writer.writeOptionalAttribute(prop.uniform); } ::std::optional<::mlir::gpu::AllReduceOperation> AllReduceOp::getOp() { auto attr = getOpAttr(); return attr ? ::std::optional<::mlir::gpu::AllReduceOperation>(attr.getValue()) : (::std::nullopt); } bool AllReduceOp::getUniform() { auto attr = getUniformAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } void AllReduceOp::setOp(::std::optional<::mlir::gpu::AllReduceOperation> attrValue) { auto &odsProp = getProperties().op; if (attrValue) odsProp = ::mlir::gpu::AllReduceOperationAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void AllReduceOp::setUniform(bool attrValue) { auto &odsProp = getProperties().uniform; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = uniform; } (void)odsState.addRegion(); odsState.addTypes(result); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { return create(builder, builder.getLoc(), std::forward(result), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = uniform; } (void)odsState.addRegion(); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(AllReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = uniform; } (void)odsState.addRegion(); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } (void)odsState.addRegion(); odsState.addTypes(result); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { return create(builder, builder.getLoc(), std::forward(result), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } (void)odsState.addRegion(); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(AllReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { odsState.addOperands(value); if (op) { odsState.getOrAddProperties().op = op; } if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } (void)odsState.addRegion(); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, /*optional*/::mlir::gpu::AllReduceOperationAttr op, /*optional*/bool uniform) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform)); } void AllReduceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); for (unsigned i = 0; i != 1; ++i) (void)odsState.addRegion(); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); for (unsigned i = 0; i != 1; ++i) (void)odsState.addRegion(); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(AllReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void AllReduceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); for (unsigned i = 0; i != 1; ++i) (void)odsState.addRegion(); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void AllReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); for (unsigned i = 0; i != 1; ++i) (void)odsState.addRegion(); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(AllReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } AllReduceOp AllReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllReduceOp AllReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult AllReduceOp::verifyInvariantsImpl() { auto tblgen_op = getProperties().op; (void)tblgen_op; auto tblgen_uniform = getProperties().uniform; (void)tblgen_uniform; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps1(*this, tblgen_op, "op"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_uniform, "uniform"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; for (auto ®ion : ::llvm::MutableArrayRef((*this)->getRegion(0))) if (::mlir::failed(__mlir_ods_local_region_constraint_GPUOps1(*this, region, "body", index++))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult AllReduceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult AllReduceOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult AllReduceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::AllReduceOperationAttr opAttr; ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; std::unique_ptr<::mlir::Region> bodyRegion = std::make_unique<::mlir::Region>(); ::llvm::ArrayRef<::mlir::Type> allOperandTypes; ::llvm::ArrayRef<::mlir::Type> allResultTypes; { auto odsResult = parseAllReduceOperation(parser, opAttr); if (odsResult) return ::mlir::failure(); if (opAttr) result.getOrAddProperties().op = opAttr; } valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("uniform"))) { result.getOrAddProperties().uniform = parser.getBuilder().getUnitAttr(); } if (parser.parseRegion(*bodyRegion)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); ::mlir::FunctionType allOperand__allResult_functionType; if (parser.parseType(allOperand__allResult_functionType)) return ::mlir::failure(); allOperandTypes = allOperand__allResult_functionType.getInputs(); allResultTypes = allOperand__allResult_functionType.getResults(); result.addRegion(std::move(bodyRegion)); result.addTypes(allResultTypes); if (parser.resolveOperands(valueOperands, allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } void AllReduceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAllReduceOperation(_odsPrinter, *this, getOpAttr()); _odsPrinter << ' '; _odsPrinter << getValue(); if ((getUniformAttr() && getUniformAttr() != ((false) ? ::mlir::OpBuilder((*this)->getContext()).getUnitAttr() : nullptr))) { _odsPrinter << ' ' << "uniform"; } _odsPrinter << ' '; _odsPrinter.printRegion(getBody()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("op"); elidedAttrs.push_back("uniform"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getUniformAttr(); if(attr && (attr == ((false) ? odsBuilder.getUnitAttr() : nullptr))) elidedAttrs.push_back("uniform"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printFunctionalType(getOperation()->getOperandTypes(), getOperation()->getResultTypes()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::AllReduceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::AllReduceOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::AllocOp definitions //===----------------------------------------------------------------------===// namespace detail { AllocOpGenericAdaptorBase::AllocOpGenericAdaptorBase(AllocOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair AllocOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::UnitAttr AllocOpGenericAdaptorBase::getHostSharedAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().hostShared); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool AllocOpGenericAdaptorBase::getHostShared() { auto attr = getHostSharedAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } } // namespace detail AllocOpAdaptor::AllocOpAdaptor(AllocOp op) : AllocOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult AllocOpAdaptor::verify(::mlir::Location loc) { auto tblgen_hostShared = getProperties().hostShared; (void)tblgen_hostShared; if (tblgen_hostShared && !((::llvm::isa<::mlir::UnitAttr>(tblgen_hostShared)))) return emitError(loc, "'gpu.alloc' op attribute 'hostShared' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } void AllocOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "memref"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair AllocOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange AllocOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(0u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange AllocOp::getDynamicSizesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange AllocOp::getSymbolOperandsMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } std::pair AllocOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult AllocOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.hostShared; auto attr = dict.get("hostShared"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `hostShared` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute AllocOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.hostShared; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("hostShared", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code AllocOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.hostShared.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional AllocOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "hostShared") return prop.hostShared; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void AllocOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "hostShared") { prop.hostShared = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void AllocOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.hostShared) attrs.append("hostShared", prop.hostShared); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult AllocOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getHostSharedAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "hostShared", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult AllocOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.hostShared))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void AllocOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.hostShared); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } bool AllocOp::getHostShared() { auto attr = getHostSharedAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } void AllocOp::setHostShared(bool attrValue) { auto &odsProp = getProperties().hostShared; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void AllocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { odsState.addOperands(asyncDependencies); odsState.addOperands(dynamicSizes); odsState.addOperands(symbolOperands); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), static_cast(dynamicSizes.size()), static_cast(symbolOperands.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (hostShared) { odsState.getOrAddProperties().hostShared = hostShared; } odsState.addTypes(memref); if (asyncToken) odsState.addTypes(asyncToken); } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(memref), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { return create(builder, builder.getLoc(), std::forward(memref), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); } void AllocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { odsState.addOperands(asyncDependencies); odsState.addOperands(dynamicSizes); odsState.addOperands(symbolOperands); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), static_cast(dynamicSizes.size()), static_cast(symbolOperands.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (hostShared) { odsState.getOrAddProperties().hostShared = hostShared; } assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/::mlir::UnitAttr hostShared) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); } void AllocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { odsState.addOperands(asyncDependencies); odsState.addOperands(dynamicSizes); odsState.addOperands(symbolOperands); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), static_cast(dynamicSizes.size()), static_cast(symbolOperands.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (hostShared) { odsState.getOrAddProperties().hostShared = ((hostShared) ? odsBuilder.getUnitAttr() : nullptr); } odsState.addTypes(memref); if (asyncToken) odsState.addTypes(asyncToken); } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(memref), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type memref, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { return create(builder, builder.getLoc(), std::forward(memref), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); } void AllocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { odsState.addOperands(asyncDependencies); odsState.addOperands(dynamicSizes); odsState.addOperands(symbolOperands); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), static_cast(dynamicSizes.size()), static_cast(symbolOperands.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (hostShared) { odsState.getOrAddProperties().hostShared = ((hostShared) ? odsBuilder.getUnitAttr() : nullptr); } assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::ValueRange dynamicSizes, ::mlir::ValueRange symbolOperands, /*optional*/bool hostShared) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dynamicSizes), std::forward(symbolOperands), std::forward(hostShared)); } void AllocOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void AllocOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } AllocOp AllocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocOp AllocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult AllocOp::verifyInvariantsImpl() { auto tblgen_hostShared = getProperties().hostShared; (void)tblgen_hostShared; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_hostShared, "hostShared"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult AllocOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult AllocOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> dynamicSizesOperands; ::llvm::SMLoc dynamicSizesOperandsLoc; (void)dynamicSizesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> symbolOperandsOperands; ::llvm::SMLoc symbolOperandsOperandsLoc; (void)symbolOperandsOperandsLoc; ::mlir::Type memrefRawType{}; ::llvm::ArrayRef<::mlir::Type> memrefTypes(&memrefRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } if (::mlir::succeeded(parser.parseOptionalKeyword("host_shared"))) { result.getOrAddProperties().hostShared = parser.getBuilder().getUnitAttr(); } if (parser.parseLParen()) return ::mlir::failure(); dynamicSizesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(dynamicSizesOperands)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLSquare())) { symbolOperandsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(symbolOperandsOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); memrefRawType = type; } ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependenciesOperands.size()), static_cast(dynamicSizesOperands.size()), static_cast(symbolOperandsOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIndexType(); result.addTypes(memrefTypes); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dynamicSizesOperands, odsBuildableType1, dynamicSizesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(symbolOperandsOperands, odsBuildableType1, symbolOperandsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void AllocOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); if ((getHostSharedAttr() && getHostSharedAttr() != ((false) ? ::mlir::OpBuilder((*this)->getContext()).getUnitAttr() : nullptr))) { _odsPrinter << ' '; _odsPrinter << "host_shared"; } _odsPrinter << ' '; _odsPrinter << "("; _odsPrinter << getDynamicSizes(); _odsPrinter << ")"; if (!getSymbolOperands().empty()) { _odsPrinter << "["; _odsPrinter << getSymbolOperands(); _odsPrinter << "]"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("hostShared"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getHostSharedAttr(); if(attr && (attr == ((false) ? odsBuilder.getUnitAttr() : nullptr))) elidedAttrs.push_back("hostShared"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void AllocOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSResultIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Allocate::get(), getOperation()->getOpResult(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::AllocOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::AllocOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::BallotOp definitions //===----------------------------------------------------------------------===// namespace detail { BallotOpGenericAdaptorBase::BallotOpGenericAdaptorBase(BallotOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail BallotOpAdaptor::BallotOpAdaptor(BallotOp op) : BallotOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult BallotOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult BallotOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute BallotOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code BallotOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional BallotOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void BallotOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void BallotOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult BallotOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void BallotOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value predicate) { odsState.addOperands(predicate); odsState.addTypes(result); } BallotOp BallotOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BallotOp BallotOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(result), std::forward(predicate)); } void BallotOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value predicate) { odsState.addOperands(predicate); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } BallotOp BallotOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value predicate) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(predicate)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BallotOp BallotOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value predicate) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(predicate)); } void BallotOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } BallotOp BallotOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BallotOp BallotOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void BallotOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } BallotOp BallotOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BallotOp BallotOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult BallotOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps7(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult BallotOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult BallotOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand predicateRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> predicateOperands(&predicateRawOperand, 1); ::llvm::SMLoc predicateOperandsLoc; (void)predicateOperandsLoc; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); predicateOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(predicateRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IntegerType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); result.addTypes(resultTypes); if (parser.resolveOperands(predicateOperands, odsBuildableType0, predicateOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void BallotOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getPredicate(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void BallotOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::BallotOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::BarrierOp definitions //===----------------------------------------------------------------------===// namespace detail { BarrierOpGenericAdaptorBase::BarrierOpGenericAdaptorBase(BarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair BarrierOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::std::optional< ::mlir::ArrayAttr > BarrierOpGenericAdaptorBase::getAddressSpaces() { auto attr = getAddressSpacesAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::mlir::gpu::BarrierScopeAttr BarrierOpGenericAdaptorBase::getScopeAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::BarrierScopeAttr>(getProperties().scope); return attr; } ::mlir::gpu::BarrierScope BarrierOpGenericAdaptorBase::getScope() { auto attr = getScopeAttr(); return attr.getValue(); } } // namespace detail BarrierOpAdaptor::BarrierOpAdaptor(BarrierOp op) : BarrierOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult BarrierOpAdaptor::verify(::mlir::Location loc) { auto tblgen_address_spaces = getProperties().address_spaces; (void)tblgen_address_spaces; auto tblgen_scope = getProperties().scope; (void)tblgen_scope; if (tblgen_address_spaces && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_address_spaces))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_address_spaces), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::gpu::AddressSpaceAttr>(attr))); })))) return emitError(loc, "'gpu.barrier' op attribute 'address_spaces' failed to satisfy constraint: GPU Address Space array"); if (tblgen_scope && !((::llvm::isa<::mlir::gpu::BarrierScopeAttr>(tblgen_scope)))) return emitError(loc, "'gpu.barrier' op attribute 'scope' failed to satisfy constraint: barrier execution scope"); return ::mlir::success(); } std::pair BarrierOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange BarrierOp::getNamedBarrierMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult BarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.address_spaces; auto attr = dict.get("address_spaces"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `address_spaces` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.scope; auto attr = dict.get("scope"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `scope` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute BarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.address_spaces; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("address_spaces", propStorage)); } { const auto &propStorage = prop.scope; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("scope", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code BarrierOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.address_spaces.getAsOpaquePointer()), llvm::hash_value(prop.scope.getAsOpaquePointer())); } std::optional BarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "address_spaces") return prop.address_spaces; if (name == "scope") return prop.scope; return std::nullopt; } void BarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "address_spaces") { prop.address_spaces = ::llvm::dyn_cast_or_null>(value); return; } if (name == "scope") { prop.scope = ::llvm::dyn_cast_or_null>(value); return; } } void BarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.address_spaces) attrs.append("address_spaces", prop.address_spaces); if (prop.scope) attrs.append("scope", prop.scope); } ::llvm::LogicalResult BarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getAddressSpacesAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps3(attr, "address_spaces", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getScopeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps4(attr, "scope", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult BarrierOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.address_spaces))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.scope))) return ::mlir::failure(); return ::mlir::success(); } void BarrierOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.address_spaces); writer.writeOptionalAttribute(prop.scope); } ::std::optional< ::mlir::ArrayAttr > BarrierOp::getAddressSpaces() { auto attr = getAddressSpacesAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::mlir::gpu::BarrierScope BarrierOp::getScope() { auto attr = getScopeAttr(); return attr.getValue(); } void BarrierOp::setScope(::mlir::gpu::BarrierScope attrValue) { getProperties().scope = ::mlir::gpu::BarrierScopeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, std::optional<::mlir::gpu::AddressSpace> addressSpace) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(addressSpace)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, std::optional<::mlir::gpu::AddressSpace> addressSpace) { return create(builder, builder.getLoc(), std::forward(addressSpace)); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value memrefToFence) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(memrefToFence)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, Value memrefToFence) { return create(builder, builder.getLoc(), std::forward(memrefToFence)); } void BarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { if (named_barrier) odsState.addOperands(named_barrier); if (address_spaces) { odsState.getOrAddProperties().address_spaces = address_spaces; } if (scope) { odsState.getOrAddProperties().scope = scope; } } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { return create(builder, builder.getLoc(), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); } void BarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { if (named_barrier) odsState.addOperands(named_barrier); if (address_spaces) { odsState.getOrAddProperties().address_spaces = address_spaces; } if (scope) { odsState.getOrAddProperties().scope = scope; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScopeAttr scope) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); } void BarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { if (named_barrier) odsState.addOperands(named_barrier); if (address_spaces) { odsState.getOrAddProperties().address_spaces = address_spaces; } odsState.getOrAddProperties().scope = ::mlir::gpu::BarrierScopeAttr::get(odsBuilder.getContext(), scope); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { return create(builder, builder.getLoc(), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); } void BarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { if (named_barrier) odsState.addOperands(named_barrier); if (address_spaces) { odsState.getOrAddProperties().address_spaces = address_spaces; } odsState.getOrAddProperties().scope = ::mlir::gpu::BarrierScopeAttr::get(odsBuilder.getContext(), scope); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::ArrayAttr address_spaces, /*optional*/::mlir::Value named_barrier, ::mlir::gpu::BarrierScope scope) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(address_spaces), std::forward(named_barrier), std::forward(scope)); } void BarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void BarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } BarrierOp BarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BarrierOp BarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void BarrierOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.scope) properties.scope = ::mlir::gpu::BarrierScopeAttr::get(odsBuilder.getContext(), ::mlir::gpu::BarrierScope::Workgroup); } ::llvm::LogicalResult BarrierOp::verifyInvariantsImpl() { auto tblgen_address_spaces = getProperties().address_spaces; (void)tblgen_address_spaces; auto tblgen_scope = getProperties().scope; (void)tblgen_scope; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps3(*this, tblgen_address_spaces, "address_spaces"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps4(*this, tblgen_scope, "scope"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); if (valueGroup0.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult BarrierOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult BarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> named_barrierOperands; ::llvm::SMLoc named_barrierOperandsLoc; (void)named_barrierOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> named_barrierTypes; ::mlir::ArrayAttr address_spacesAttr; ::mlir::gpu::BarrierScopeAttr scopeAttr; bool namedClause = false; bool memfenceClause = false; bool scopeClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("named"))) { if (namedClause) { return parser.emitError(parser.getNameLoc()) << "`named` clause can appear at most once in the expansion of the " "oilist directive"; } namedClause = true; if (parser.parseLParen()) return ::mlir::failure(); { named_barrierOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); named_barrierOperands.push_back(operand); } } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type optionalType; ::mlir::OptionalParseResult parseResult = parser.parseOptionalType(optionalType); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); named_barrierTypes.push_back(optionalType); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("memfence"))) { if (memfenceClause) { return parser.emitError(parser.getNameLoc()) << "`memfence` clause can appear at most once in the expansion of the " "oilist directive"; } memfenceClause = true; ::mlir::OptionalParseResult parseResultaddress_spacesAttr = parser.parseOptionalAttribute(address_spacesAttr, parser.getBuilder().getType<::mlir::NoneType>()); if (parseResultaddress_spacesAttr.has_value() && failed(*parseResultaddress_spacesAttr)) return ::mlir::failure(); if (parseResultaddress_spacesAttr.has_value() && succeeded(*parseResultaddress_spacesAttr)) if (address_spacesAttr) result.getOrAddProperties().address_spaces = address_spacesAttr; } else if (succeeded(parser.parseOptionalKeyword("scope"))) { if (scopeClause) { return parser.emitError(parser.getNameLoc()) << "`scope` clause can appear at most once in the expansion of the " "oilist directive"; } scopeClause = true; if (parser.parseCustomAttributeWithFallback(scopeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (scopeAttr) result.getOrAddProperties().scope = scopeAttr; } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (!named_barrierOperands.empty()) { if (parser.resolveOperands(named_barrierOperands, named_barrierTypes, named_barrierOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void BarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) { if (false || getNamedBarrier()) { _odsPrinter << ' ' << "named"; _odsPrinter << "("; if (::mlir::Value value = getNamedBarrier()) _odsPrinter << value; _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << (getNamedBarrier() ? ::llvm::ArrayRef<::mlir::Type>(getNamedBarrier().getType()) : ::llvm::ArrayRef<::mlir::Type>()); _odsPrinter << ")"; } if (false || (getAddressSpacesAttr())) { _odsPrinter << ' ' << "memfence"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getAddressSpacesAttr()); } if (false || (getScopeAttr() != ::mlir::gpu::BarrierScopeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), ::mlir::gpu::BarrierScope::Workgroup))) { _odsPrinter << ' ' << "scope"; _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getScopeAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("address_spaces"); elidedAttrs.push_back("scope"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getScopeAttr(); if(attr && (attr == ::mlir::gpu::BarrierScopeAttr::get(odsBuilder.getContext(), ::mlir::gpu::BarrierScope::Workgroup))) elidedAttrs.push_back("scope"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::BarrierOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::BarrierOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::BinaryOp definitions //===----------------------------------------------------------------------===// namespace detail { BinaryOpGenericAdaptorBase::BinaryOpGenericAdaptorBase(BinaryOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::llvm::StringRef BinaryOpGenericAdaptorBase::getSymName() { auto attr = getSymNameAttr(); return attr.getValue(); } ::std::optional<::mlir::Attribute> BinaryOpGenericAdaptorBase::getOffloadingHandler() { auto attr = getOffloadingHandlerAttr(); return attr ? ::std::optional<::mlir::Attribute>(attr) : (::std::nullopt); } ::mlir::ArrayAttr BinaryOpGenericAdaptorBase::getObjects() { auto attr = getObjectsAttr(); return attr; } } // namespace detail BinaryOpAdaptor::BinaryOpAdaptor(BinaryOp op) : BinaryOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult BinaryOpAdaptor::verify(::mlir::Location loc) { auto tblgen_objects = getProperties().objects; (void)tblgen_objects; if (!tblgen_objects) return emitError(loc, "'gpu.binary' op requires attribute 'objects'"); auto tblgen_offloadingHandler = getProperties().offloadingHandler; (void)tblgen_offloadingHandler; auto tblgen_sym_name = getProperties().sym_name; (void)tblgen_sym_name; if (!tblgen_sym_name) return emitError(loc, "'gpu.binary' op requires attribute 'sym_name'"); if (tblgen_sym_name && !((::llvm::isa<::mlir::StringAttr>(tblgen_sym_name)))) return emitError(loc, "'gpu.binary' op attribute 'sym_name' failed to satisfy constraint: string attribute"); if (tblgen_offloadingHandler && !(((true)) && ((tblgen_offloadingHandler.hasTrait<::mlir::gpu::OffloadingTranslationAttrTrait>())))) return emitError(loc, "'gpu.binary' op attribute 'offloadingHandler' failed to satisfy constraint: any attribute with the `OffloadingTranslationAttrTrait` trait."); if (tblgen_objects && !((((::llvm::isa<::mlir::ArrayAttr>(tblgen_objects))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_objects), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::gpu::ObjectAttr>(attr))); }))) && ((::llvm::cast<::mlir::ArrayAttr>(tblgen_objects).size() >= 1)))) return emitError(loc, "'gpu.binary' op attribute 'objects' failed to satisfy constraint: an array of GPU object attributes with at least 1 elements"); return ::mlir::success(); } ::llvm::LogicalResult BinaryOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.objects; auto attr = dict.get("objects"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `objects` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.offloadingHandler; auto attr = dict.get("offloadingHandler"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `offloadingHandler` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.sym_name; auto attr = dict.get("sym_name"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `sym_name` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute BinaryOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.objects; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("objects", propStorage)); } { const auto &propStorage = prop.offloadingHandler; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("offloadingHandler", propStorage)); } { const auto &propStorage = prop.sym_name; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("sym_name", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code BinaryOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.objects.getAsOpaquePointer()), llvm::hash_value(prop.offloadingHandler.getAsOpaquePointer()), llvm::hash_value(prop.sym_name.getAsOpaquePointer())); } std::optional BinaryOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "objects") return prop.objects; if (name == "offloadingHandler") return prop.offloadingHandler; if (name == "sym_name") return prop.sym_name; return std::nullopt; } void BinaryOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "objects") { prop.objects = ::llvm::dyn_cast_or_null>(value); return; } if (name == "offloadingHandler") { prop.offloadingHandler = ::llvm::dyn_cast_or_null>(value); return; } if (name == "sym_name") { prop.sym_name = ::llvm::dyn_cast_or_null>(value); return; } } void BinaryOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.objects) attrs.append("objects", prop.objects); if (prop.offloadingHandler) attrs.append("offloadingHandler", prop.offloadingHandler); if (prop.sym_name) attrs.append("sym_name", prop.sym_name); } ::llvm::LogicalResult BinaryOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getObjectsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps7(attr, "objects", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getOffloadingHandlerAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps6(attr, "offloadingHandler", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getSymNameAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(attr, "sym_name", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult BinaryOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.objects))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.offloadingHandler))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.sym_name))) return ::mlir::failure(); return ::mlir::success(); } void BinaryOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.objects); writer.writeOptionalAttribute(prop.offloadingHandler); writer.writeAttribute(prop.sym_name); } ::llvm::StringRef BinaryOp::getSymName() { auto attr = getSymNameAttr(); return attr.getValue(); } ::std::optional<::mlir::Attribute> BinaryOp::getOffloadingHandler() { auto attr = getOffloadingHandlerAttr(); return attr ? ::std::optional<::mlir::Attribute>(attr) : (::std::nullopt); } ::mlir::ArrayAttr BinaryOp::getObjects() { auto attr = getObjectsAttr(); return attr; } void BinaryOp::setSymName(::llvm::StringRef attrValue) { getProperties().sym_name = ::mlir::Builder((*this)->getContext()).getStringAttr(attrValue); } BinaryOp BinaryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, StringRef name, Attribute offloadingHandler, ArrayAttr objects) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(name), std::forward(offloadingHandler), std::forward(objects)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BinaryOp BinaryOp::create(::mlir::ImplicitLocOpBuilder &builder, StringRef name, Attribute offloadingHandler, ArrayAttr objects) { return create(builder, builder.getLoc(), std::forward(name), std::forward(offloadingHandler), std::forward(objects)); } BinaryOp BinaryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, StringRef name, Attribute offloadingHandler, ArrayRef objects) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(name), std::forward(offloadingHandler), std::forward(objects)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BinaryOp BinaryOp::create(::mlir::ImplicitLocOpBuilder &builder, StringRef name, Attribute offloadingHandler, ArrayRef objects) { return create(builder, builder.getLoc(), std::forward(name), std::forward(offloadingHandler), std::forward(objects)); } ::llvm::LogicalResult BinaryOp::verifyInvariantsImpl() { auto tblgen_objects = getProperties().objects; (void)tblgen_objects; if (!tblgen_objects) return emitOpError("requires attribute 'objects'"); auto tblgen_offloadingHandler = getProperties().offloadingHandler; (void)tblgen_offloadingHandler; auto tblgen_sym_name = getProperties().sym_name; (void)tblgen_sym_name; if (!tblgen_sym_name) return emitOpError("requires attribute 'sym_name'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(*this, tblgen_sym_name, "sym_name"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps6(*this, tblgen_offloadingHandler, "offloadingHandler"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps7(*this, tblgen_objects, "objects"))) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult BinaryOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult BinaryOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::StringAttr sym_nameAttr; ::mlir::Attribute offloadingHandlerAttr; ::mlir::ArrayAttr objectsAttr; if (parser.parseSymbolName(sym_nameAttr)) return ::mlir::failure(); if (sym_nameAttr) result.getOrAddProperties().sym_name = sym_nameAttr; { auto odsResult = parseOffloadingHandler(parser, offloadingHandlerAttr); if (odsResult) return ::mlir::failure(); if (offloadingHandlerAttr) result.getOrAddProperties().offloadingHandler = offloadingHandlerAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseCustomAttributeWithFallback(objectsAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (objectsAttr) result.getOrAddProperties().objects = objectsAttr; return ::mlir::success(); } void BinaryOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printSymbolName(getSymNameAttr().getValue()); _odsPrinter << ' '; printOffloadingHandler(_odsPrinter, *this, getOffloadingHandlerAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("sym_name"); elidedAttrs.push_back("offloadingHandler"); elidedAttrs.push_back("objects"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getObjectsAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::BinaryOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::BinaryOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::BlockDimOp definitions //===----------------------------------------------------------------------===// namespace detail { BlockDimOpGenericAdaptorBase::BlockDimOpGenericAdaptorBase(BlockDimOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension BlockDimOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > BlockDimOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail BlockDimOpAdaptor::BlockDimOpAdaptor(BlockDimOp op) : BlockDimOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult BlockDimOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.block_dim' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.block_dim' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.block_dim' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult BlockDimOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute BlockDimOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code BlockDimOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional BlockDimOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void BlockDimOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void BlockDimOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult BlockDimOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult BlockDimOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void BlockDimOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension BlockDimOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > BlockDimOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void BlockDimOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void BlockDimOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void BlockDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void BlockDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void BlockDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } BlockDimOp BlockDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockDimOp BlockDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult BlockDimOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult BlockDimOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult BlockDimOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult BlockDimOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void BlockDimOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void BlockDimOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void BlockDimOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::BlockDimOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::BlockDimOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::BlockIdOp definitions //===----------------------------------------------------------------------===// namespace detail { BlockIdOpGenericAdaptorBase::BlockIdOpGenericAdaptorBase(BlockIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension BlockIdOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > BlockIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail BlockIdOpAdaptor::BlockIdOpAdaptor(BlockIdOp op) : BlockIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult BlockIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.block_id' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.block_id' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.block_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult BlockIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute BlockIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code BlockIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional BlockIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void BlockIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void BlockIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult BlockIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult BlockIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void BlockIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension BlockIdOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > BlockIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void BlockIdOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void BlockIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void BlockIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void BlockIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void BlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(BlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } BlockIdOp BlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } BlockIdOp BlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult BlockIdOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult BlockIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult BlockIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult BlockIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void BlockIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void BlockIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void BlockIdOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::BlockIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::BlockIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ClusterBlockIdOp definitions //===----------------------------------------------------------------------===// namespace detail { ClusterBlockIdOpGenericAdaptorBase::ClusterBlockIdOpGenericAdaptorBase(ClusterBlockIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension ClusterBlockIdOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterBlockIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail ClusterBlockIdOpAdaptor::ClusterBlockIdOpAdaptor(ClusterBlockIdOp op) : ClusterBlockIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ClusterBlockIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.cluster_block_id' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.cluster_block_id' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.cluster_block_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult ClusterBlockIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ClusterBlockIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ClusterBlockIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional ClusterBlockIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void ClusterBlockIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void ClusterBlockIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult ClusterBlockIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ClusterBlockIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void ClusterBlockIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension ClusterBlockIdOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterBlockIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void ClusterBlockIdOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ClusterBlockIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterBlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterBlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterBlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ClusterBlockIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterBlockIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterBlockIdOp ClusterBlockIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ClusterBlockIdOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ClusterBlockIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ClusterBlockIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ClusterBlockIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void ClusterBlockIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void ClusterBlockIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void ClusterBlockIdOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ClusterBlockIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ClusterBlockIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ClusterDimBlocksOp definitions //===----------------------------------------------------------------------===// namespace detail { ClusterDimBlocksOpGenericAdaptorBase::ClusterDimBlocksOpGenericAdaptorBase(ClusterDimBlocksOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension ClusterDimBlocksOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterDimBlocksOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail ClusterDimBlocksOpAdaptor::ClusterDimBlocksOpAdaptor(ClusterDimBlocksOp op) : ClusterDimBlocksOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ClusterDimBlocksOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.cluster_dim_blocks' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.cluster_dim_blocks' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.cluster_dim_blocks' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult ClusterDimBlocksOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ClusterDimBlocksOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ClusterDimBlocksOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional ClusterDimBlocksOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void ClusterDimBlocksOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void ClusterDimBlocksOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult ClusterDimBlocksOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ClusterDimBlocksOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void ClusterDimBlocksOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension ClusterDimBlocksOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterDimBlocksOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void ClusterDimBlocksOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ClusterDimBlocksOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimBlocksOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimBlocksOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimBlocksOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ClusterDimBlocksOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimBlocksOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimBlocksOp ClusterDimBlocksOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ClusterDimBlocksOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ClusterDimBlocksOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ClusterDimBlocksOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ClusterDimBlocksOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void ClusterDimBlocksOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void ClusterDimBlocksOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void ClusterDimBlocksOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ClusterDimBlocksOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ClusterDimBlocksOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ClusterDimOp definitions //===----------------------------------------------------------------------===// namespace detail { ClusterDimOpGenericAdaptorBase::ClusterDimOpGenericAdaptorBase(ClusterDimOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension ClusterDimOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterDimOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail ClusterDimOpAdaptor::ClusterDimOpAdaptor(ClusterDimOp op) : ClusterDimOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ClusterDimOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.cluster_dim' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.cluster_dim' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.cluster_dim' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult ClusterDimOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ClusterDimOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ClusterDimOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional ClusterDimOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void ClusterDimOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void ClusterDimOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult ClusterDimOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ClusterDimOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void ClusterDimOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension ClusterDimOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterDimOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void ClusterDimOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ClusterDimOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ClusterDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ClusterDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterDimOp ClusterDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterDimOp ClusterDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ClusterDimOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ClusterDimOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ClusterDimOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ClusterDimOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void ClusterDimOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void ClusterDimOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void ClusterDimOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ClusterDimOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ClusterDimOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ClusterIdOp definitions //===----------------------------------------------------------------------===// namespace detail { ClusterIdOpGenericAdaptorBase::ClusterIdOpGenericAdaptorBase(ClusterIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension ClusterIdOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail ClusterIdOpAdaptor::ClusterIdOpAdaptor(ClusterIdOp op) : ClusterIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ClusterIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.cluster_id' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.cluster_id' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.cluster_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult ClusterIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ClusterIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ClusterIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional ClusterIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void ClusterIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void ClusterIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult ClusterIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ClusterIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void ClusterIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension ClusterIdOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ClusterIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void ClusterIdOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ClusterIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ClusterIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ClusterIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ClusterIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ClusterIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ClusterIdOp ClusterIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ClusterIdOp ClusterIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ClusterIdOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ClusterIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ClusterIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ClusterIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void ClusterIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void ClusterIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void ClusterIdOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ClusterIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ClusterIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::Create2To4SpMatOp definitions //===----------------------------------------------------------------------===// namespace detail { Create2To4SpMatOpGenericAdaptorBase::Create2To4SpMatOpGenericAdaptorBase(Create2To4SpMatOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair Create2To4SpMatOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::Prune2To4SpMatFlagAttr Create2To4SpMatOpGenericAdaptorBase::getPruneFlagAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::Prune2To4SpMatFlagAttr>(getProperties().pruneFlag); return attr; } ::mlir::gpu::Prune2To4SpMatFlag Create2To4SpMatOpGenericAdaptorBase::getPruneFlag() { auto attr = getPruneFlagAttr(); return attr.getValue(); } } // namespace detail Create2To4SpMatOpAdaptor::Create2To4SpMatOpAdaptor(Create2To4SpMatOp op) : Create2To4SpMatOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult Create2To4SpMatOpAdaptor::verify(::mlir::Location loc) { auto tblgen_pruneFlag = getProperties().pruneFlag; (void)tblgen_pruneFlag; if (tblgen_pruneFlag && !((::llvm::isa<::mlir::gpu::Prune2To4SpMatFlagAttr>(tblgen_pruneFlag)))) return emitError(loc, "'gpu.create_2to4_spmat' op attribute 'pruneFlag' failed to satisfy constraint: pruning strategy for 2:4 sparse matrix"); return ::mlir::success(); } void Create2To4SpMatOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spMat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair Create2To4SpMatOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange Create2To4SpMatOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair Create2To4SpMatOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult Create2To4SpMatOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.pruneFlag; auto attr = dict.get("pruneFlag"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `pruneFlag` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute Create2To4SpMatOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.pruneFlag; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("pruneFlag", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code Create2To4SpMatOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.pruneFlag.getAsOpaquePointer())); } std::optional Create2To4SpMatOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "pruneFlag") return prop.pruneFlag; return std::nullopt; } void Create2To4SpMatOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "pruneFlag") { prop.pruneFlag = ::llvm::dyn_cast_or_null>(value); return; } } void Create2To4SpMatOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.pruneFlag) attrs.append("pruneFlag", prop.pruneFlag); } ::llvm::LogicalResult Create2To4SpMatOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getPruneFlagAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps10(attr, "pruneFlag", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult Create2To4SpMatOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.pruneFlag))) return ::mlir::failure(); return ::mlir::success(); } void Create2To4SpMatOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.pruneFlag); } ::mlir::gpu::Prune2To4SpMatFlag Create2To4SpMatOp::getPruneFlag() { auto attr = getPruneFlagAttr(); return attr.getValue(); } void Create2To4SpMatOp::setPruneFlag(::mlir::gpu::Prune2To4SpMatFlag attrValue) { getProperties().pruneFlag = ::mlir::gpu::Prune2To4SpMatFlagAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void Create2To4SpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(memref); if (pruneFlag) { odsState.getOrAddProperties().pruneFlag = pruneFlag; } odsState.addTypes(spMat); if (asyncToken) odsState.addTypes(asyncToken); } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spMat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(spMat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); } void Create2To4SpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(memref); if (pruneFlag) { odsState.getOrAddProperties().pruneFlag = pruneFlag; } assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlag, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); } void Create2To4SpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(memref); odsState.getOrAddProperties().pruneFlag = ::mlir::gpu::Prune2To4SpMatFlagAttr::get(odsBuilder.getContext(), pruneFlag); odsState.addTypes(spMat); if (asyncToken) odsState.addTypes(asyncToken); } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spMat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spMat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(spMat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); } void Create2To4SpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(memref); odsState.getOrAddProperties().pruneFlag = ::mlir::gpu::Prune2To4SpMatFlagAttr::get(odsBuilder.getContext(), pruneFlag); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::gpu::Prune2To4SpMatFlag pruneFlag, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(pruneFlag), std::forward(memref)); } void Create2To4SpMatOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void Create2To4SpMatOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } Create2To4SpMatOp Create2To4SpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void Create2To4SpMatOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.pruneFlag) properties.pruneFlag = ::mlir::gpu::Prune2To4SpMatFlagAttr::get(odsBuilder.getContext(), Prune2To4SpMatFlag::PRUNE_AND_CHECK); } ::llvm::LogicalResult Create2To4SpMatOp::verifyInvariantsImpl() { auto tblgen_pruneFlag = getProperties().pruneFlag; (void)tblgen_pruneFlag; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps10(*this, tblgen_pruneFlag, "pruneFlag"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult Create2To4SpMatOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult Create2To4SpMatOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::gpu::Prune2To4SpMatFlagAttr pruneFlagAttr; ::mlir::OpAsmParser::UnresolvedOperand rowsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowsOperands(&rowsRawOperand, 1); ::llvm::SMLoc rowsOperandsLoc; (void)rowsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colsOperands(&colsRawOperand, 1); ::llvm::SMLoc colsOperandsLoc; (void)colsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand memrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> memrefOperands(&memrefRawOperand, 1); ::llvm::SMLoc memrefOperandsLoc; (void)memrefOperandsLoc; ::mlir::Type memrefRawType{}; ::llvm::ArrayRef<::mlir::Type> memrefTypes(&memrefRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } if (parser.parseLBrace()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(pruneFlagAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (pruneFlagAttr) result.getOrAddProperties().pruneFlag = pruneFlagAttr; if (parser.parseRBrace()) return ::mlir::failure(); rowsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); memrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(memrefRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); memrefRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowsOperands, odsBuildableType2, rowsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colsOperands, odsBuildableType2, colsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(memrefOperands, memrefTypes, memrefOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void Create2To4SpMatOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getPruneFlagAttr()); _odsPrinter << "}"; _odsPrinter << ' '; _odsPrinter << getRows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMemref(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("pruneFlag"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getPruneFlagAttr(); if(attr && (attr == ::mlir::gpu::Prune2To4SpMatFlagAttr::get(odsBuilder.getContext(), Prune2To4SpMatFlag::PRUNE_AND_CHECK))) elidedAttrs.push_back("pruneFlag"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::Create2To4SpMatOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::Create2To4SpMatOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateBsrOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateBsrOpGenericAdaptorBase::CreateBsrOpGenericAdaptorBase(CreateBsrOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateBsrOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 8) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail CreateBsrOpAdaptor::CreateBsrOpAdaptor(CreateBsrOp op) : CreateBsrOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateBsrOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateBsrOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spmat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateBsrOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 8) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange CreateBsrOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair CreateBsrOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateBsrOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateBsrOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateBsrOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateBsrOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateBsrOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateBsrOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateBsrOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateBsrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(brows); odsState.addOperands(bcols); odsState.addOperands(bnnz); odsState.addOperands(rBlockSize); odsState.addOperands(cBlockSize); odsState.addOperands(bRowPos); odsState.addOperands(bColIdxs); odsState.addOperands(values); odsState.addTypes(spmat); if (asyncToken) odsState.addTypes(asyncToken); } CreateBsrOp CreateBsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(brows), std::forward(bcols), std::forward(bnnz), std::forward(rBlockSize), std::forward(cBlockSize), std::forward(bRowPos), std::forward(bColIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateBsrOp CreateBsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(brows), std::forward(bcols), std::forward(bnnz), std::forward(rBlockSize), std::forward(cBlockSize), std::forward(bRowPos), std::forward(bColIdxs), std::forward(values)); } void CreateBsrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(brows); odsState.addOperands(bcols); odsState.addOperands(bnnz); odsState.addOperands(rBlockSize); odsState.addOperands(cBlockSize); odsState.addOperands(bRowPos); odsState.addOperands(bColIdxs); odsState.addOperands(values); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateBsrOp CreateBsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(brows), std::forward(bcols), std::forward(bnnz), std::forward(rBlockSize), std::forward(cBlockSize), std::forward(bRowPos), std::forward(bColIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateBsrOp CreateBsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value brows, ::mlir::Value bcols, ::mlir::Value bnnz, ::mlir::Value rBlockSize, ::mlir::Value cBlockSize, ::mlir::Value bRowPos, ::mlir::Value bColIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(brows), std::forward(bcols), std::forward(bnnz), std::forward(rBlockSize), std::forward(cBlockSize), std::forward(bRowPos), std::forward(bColIdxs), std::forward(values)); } void CreateBsrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 8u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateBsrOp CreateBsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateBsrOp CreateBsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateBsrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 8u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateBsrOp CreateBsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateBsrOp CreateBsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateBsrOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup7 = getODSOperands(7); for (auto v : valueGroup7) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup8 = getODSOperands(8); for (auto v : valueGroup8) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateBsrOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateBsrOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand browsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> browsOperands(&browsRawOperand, 1); ::llvm::SMLoc browsOperandsLoc; (void)browsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bcolsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bcolsOperands(&bcolsRawOperand, 1); ::llvm::SMLoc bcolsOperandsLoc; (void)bcolsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bnnzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bnnzOperands(&bnnzRawOperand, 1); ::llvm::SMLoc bnnzOperandsLoc; (void)bnnzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rBlockSizeRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rBlockSizeOperands(&rBlockSizeRawOperand, 1); ::llvm::SMLoc rBlockSizeOperandsLoc; (void)rBlockSizeOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand cBlockSizeRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> cBlockSizeOperands(&cBlockSizeRawOperand, 1); ::llvm::SMLoc cBlockSizeOperandsLoc; (void)cBlockSizeOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bRowPosRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bRowPosOperands(&bRowPosRawOperand, 1); ::llvm::SMLoc bRowPosOperandsLoc; (void)bRowPosOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bColIdxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bColIdxsOperands(&bColIdxsRawOperand, 1); ::llvm::SMLoc bColIdxsOperandsLoc; (void)bColIdxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type bRowPosRawType{}; ::llvm::ArrayRef<::mlir::Type> bRowPosTypes(&bRowPosRawType, 1); ::mlir::Type bColIdxsRawType{}; ::llvm::ArrayRef<::mlir::Type> bColIdxsTypes(&bColIdxsRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } browsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(browsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bcolsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bcolsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bnnzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bnnzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rBlockSizeOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rBlockSizeRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); cBlockSizeOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(cBlockSizeRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bRowPosOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bRowPosRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bColIdxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bColIdxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); bRowPosRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); bColIdxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(browsOperands, odsBuildableType2, browsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bcolsOperands, odsBuildableType2, bcolsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bnnzOperands, odsBuildableType2, bnnzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rBlockSizeOperands, odsBuildableType2, rBlockSizeOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(cBlockSizeOperands, odsBuildableType2, cBlockSizeOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bRowPosOperands, bRowPosTypes, bRowPosOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bColIdxsOperands, bColIdxsTypes, bColIdxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateBsrOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getBrows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBcols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBnnz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRBlockSize(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCBlockSize(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBRowPos(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBColIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBRowPos().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getBColIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateBsrOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateCooAoSOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateCooAoSOpGenericAdaptorBase::CreateCooAoSOpGenericAdaptorBase(CreateCooAoSOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateCooAoSOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 5) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail CreateCooAoSOpAdaptor::CreateCooAoSOpAdaptor(CreateCooAoSOp op) : CreateCooAoSOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateCooAoSOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateCooAoSOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spmat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateCooAoSOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 5) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange CreateCooAoSOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair CreateCooAoSOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateCooAoSOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateCooAoSOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateCooAoSOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateCooAoSOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateCooAoSOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateCooAoSOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateCooAoSOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateCooAoSOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(idxs); odsState.addOperands(values); odsState.addTypes(spmat); if (asyncToken) odsState.addTypes(asyncToken); } CreateCooAoSOp CreateCooAoSOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(idxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooAoSOp CreateCooAoSOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(idxs), std::forward(values)); } void CreateCooAoSOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(idxs); odsState.addOperands(values); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateCooAoSOp CreateCooAoSOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(idxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooAoSOp CreateCooAoSOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value idxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(idxs), std::forward(values)); } void CreateCooAoSOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 5u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCooAoSOp CreateCooAoSOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooAoSOp CreateCooAoSOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateCooAoSOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 5u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCooAoSOp CreateCooAoSOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooAoSOp CreateCooAoSOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateCooAoSOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateCooAoSOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateCooAoSOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowsOperands(&rowsRawOperand, 1); ::llvm::SMLoc rowsOperandsLoc; (void)rowsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colsOperands(&colsRawOperand, 1); ::llvm::SMLoc colsOperandsLoc; (void)colsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand nnzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nnzOperands(&nnzRawOperand, 1); ::llvm::SMLoc nnzOperandsLoc; (void)nnzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand idxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> idxsOperands(&idxsRawOperand, 1); ::llvm::SMLoc idxsOperandsLoc; (void)idxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type idxsRawType{}; ::llvm::ArrayRef<::mlir::Type> idxsTypes(&idxsRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } rowsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); nnzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nnzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); idxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(idxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); idxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowsOperands, odsBuildableType2, rowsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colsOperands, odsBuildableType2, colsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(nnzOperands, odsBuildableType2, nnzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(idxsOperands, idxsTypes, idxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateCooAoSOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getRows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getNnz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateCooAoSOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateCooOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateCooOpGenericAdaptorBase::CreateCooOpGenericAdaptorBase(CreateCooOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateCooOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail CreateCooOpAdaptor::CreateCooOpAdaptor(CreateCooOp op) : CreateCooOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateCooOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateCooOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spmat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateCooOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange CreateCooOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair CreateCooOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateCooOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateCooOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateCooOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateCooOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateCooOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateCooOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateCooOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateCooOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(rowIdxs); odsState.addOperands(colIdxs); odsState.addOperands(values); odsState.addTypes(spmat); if (asyncToken) odsState.addTypes(asyncToken); } CreateCooOp CreateCooOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowIdxs), std::forward(colIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooOp CreateCooOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowIdxs), std::forward(colIdxs), std::forward(values)); } void CreateCooOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(rowIdxs); odsState.addOperands(colIdxs); odsState.addOperands(values); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateCooOp CreateCooOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowIdxs), std::forward(colIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooOp CreateCooOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowIdxs, ::mlir::Value colIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowIdxs), std::forward(colIdxs), std::forward(values)); } void CreateCooOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCooOp CreateCooOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooOp CreateCooOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateCooOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCooOp CreateCooOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCooOp CreateCooOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateCooOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateCooOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateCooOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowsOperands(&rowsRawOperand, 1); ::llvm::SMLoc rowsOperandsLoc; (void)rowsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colsOperands(&colsRawOperand, 1); ::llvm::SMLoc colsOperandsLoc; (void)colsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand nnzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nnzOperands(&nnzRawOperand, 1); ::llvm::SMLoc nnzOperandsLoc; (void)nnzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowIdxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowIdxsOperands(&rowIdxsRawOperand, 1); ::llvm::SMLoc rowIdxsOperandsLoc; (void)rowIdxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colIdxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colIdxsOperands(&colIdxsRawOperand, 1); ::llvm::SMLoc colIdxsOperandsLoc; (void)colIdxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type rowIdxsRawType{}; ::llvm::ArrayRef<::mlir::Type> rowIdxsTypes(&rowIdxsRawType, 1); ::mlir::Type colIdxsRawType{}; ::llvm::ArrayRef<::mlir::Type> colIdxsTypes(&colIdxsRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } rowsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); nnzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nnzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rowIdxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowIdxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colIdxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colIdxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rowIdxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); colIdxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowsOperands, odsBuildableType2, rowsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colsOperands, odsBuildableType2, colsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(nnzOperands, odsBuildableType2, nnzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowIdxsOperands, rowIdxsTypes, rowIdxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colIdxsOperands, colIdxsTypes, colIdxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateCooOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getRows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getNnz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRowIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getColIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getRowIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getColIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateCooOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateCscOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateCscOpGenericAdaptorBase::CreateCscOpGenericAdaptorBase(CreateCscOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateCscOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail CreateCscOpAdaptor::CreateCscOpAdaptor(CreateCscOp op) : CreateCscOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateCscOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateCscOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spmat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateCscOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange CreateCscOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair CreateCscOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateCscOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateCscOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateCscOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateCscOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateCscOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateCscOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateCscOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateCscOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(colPos); odsState.addOperands(rowIdxs); odsState.addOperands(values); odsState.addTypes(spmat); if (asyncToken) odsState.addTypes(asyncToken); } CreateCscOp CreateCscOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(colPos), std::forward(rowIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCscOp CreateCscOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(colPos), std::forward(rowIdxs), std::forward(values)); } void CreateCscOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(colPos); odsState.addOperands(rowIdxs); odsState.addOperands(values); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateCscOp CreateCscOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(colPos), std::forward(rowIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCscOp CreateCscOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value colPos, ::mlir::Value rowIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(colPos), std::forward(rowIdxs), std::forward(values)); } void CreateCscOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCscOp CreateCscOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCscOp CreateCscOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateCscOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCscOp CreateCscOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCscOp CreateCscOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateCscOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateCscOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateCscOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowsOperands(&rowsRawOperand, 1); ::llvm::SMLoc rowsOperandsLoc; (void)rowsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colsOperands(&colsRawOperand, 1); ::llvm::SMLoc colsOperandsLoc; (void)colsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand nnzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nnzOperands(&nnzRawOperand, 1); ::llvm::SMLoc nnzOperandsLoc; (void)nnzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colPosRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colPosOperands(&colPosRawOperand, 1); ::llvm::SMLoc colPosOperandsLoc; (void)colPosOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowIdxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowIdxsOperands(&rowIdxsRawOperand, 1); ::llvm::SMLoc rowIdxsOperandsLoc; (void)rowIdxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type colPosRawType{}; ::llvm::ArrayRef<::mlir::Type> colPosTypes(&colPosRawType, 1); ::mlir::Type rowIdxsRawType{}; ::llvm::ArrayRef<::mlir::Type> rowIdxsTypes(&rowIdxsRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } rowsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); nnzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nnzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colPosOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colPosRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rowIdxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowIdxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); colPosRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rowIdxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowsOperands, odsBuildableType2, rowsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colsOperands, odsBuildableType2, colsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(nnzOperands, odsBuildableType2, nnzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colPosOperands, colPosTypes, colPosOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowIdxsOperands, rowIdxsTypes, rowIdxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateCscOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getRows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getNnz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getColPos(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRowIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getColPos().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRowIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateCscOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateCsrOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateCsrOpGenericAdaptorBase::CreateCsrOpGenericAdaptorBase(CreateCsrOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateCsrOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail CreateCsrOpAdaptor::CreateCsrOpAdaptor(CreateCsrOp op) : CreateCsrOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateCsrOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateCsrOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "spmat"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateCsrOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange CreateCsrOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair CreateCsrOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateCsrOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateCsrOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateCsrOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateCsrOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateCsrOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateCsrOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateCsrOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateCsrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(rowPos); odsState.addOperands(colIdxs); odsState.addOperands(values); odsState.addTypes(spmat); if (asyncToken) odsState.addTypes(asyncToken); } CreateCsrOp CreateCsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowPos), std::forward(colIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCsrOp CreateCsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type spmat, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(spmat), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowPos), std::forward(colIdxs), std::forward(values)); } void CreateCsrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(rows); odsState.addOperands(cols); odsState.addOperands(nnz); odsState.addOperands(rowPos); odsState.addOperands(colIdxs); odsState.addOperands(values); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateCsrOp CreateCsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowPos), std::forward(colIdxs), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCsrOp CreateCsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value rows, ::mlir::Value cols, ::mlir::Value nnz, ::mlir::Value rowPos, ::mlir::Value colIdxs, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(rowPos), std::forward(colIdxs), std::forward(values)); } void CreateCsrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCsrOp CreateCsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCsrOp CreateCsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateCsrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateCsrOp CreateCsrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateCsrOp CreateCsrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateCsrOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateCsrOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateCsrOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowsOperands(&rowsRawOperand, 1); ::llvm::SMLoc rowsOperandsLoc; (void)rowsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colsOperands(&colsRawOperand, 1); ::llvm::SMLoc colsOperandsLoc; (void)colsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand nnzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nnzOperands(&nnzRawOperand, 1); ::llvm::SMLoc nnzOperandsLoc; (void)nnzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rowPosRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rowPosOperands(&rowPosRawOperand, 1); ::llvm::SMLoc rowPosOperandsLoc; (void)rowPosOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand colIdxsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> colIdxsOperands(&colIdxsRawOperand, 1); ::llvm::SMLoc colIdxsOperandsLoc; (void)colIdxsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type rowPosRawType{}; ::llvm::ArrayRef<::mlir::Type> rowPosTypes(&rowPosRawType, 1); ::mlir::Type colIdxsRawType{}; ::llvm::ArrayRef<::mlir::Type> colIdxsTypes(&colIdxsRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } rowsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); nnzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nnzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rowPosOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rowPosRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); colIdxsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(colIdxsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rowPosRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); colIdxsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowsOperands, odsBuildableType2, rowsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colsOperands, odsBuildableType2, colsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(nnzOperands, odsBuildableType2, nnzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rowPosOperands, rowPosTypes, rowPosOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(colIdxsOperands, colIdxsTypes, colIdxsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateCsrOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getRows(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCols(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getNnz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRowPos(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getColIdxs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getRowPos().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getColIdxs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateCsrOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::CreateDnTensorOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateDnTensorOpGenericAdaptorBase::CreateDnTensorOpGenericAdaptorBase(CreateDnTensorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateDnTensorOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail CreateDnTensorOpAdaptor::CreateDnTensorOpAdaptor(CreateDnTensorOp op) : CreateDnTensorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateDnTensorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void CreateDnTensorOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "dnTensor"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair CreateDnTensorOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange CreateDnTensorOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(0u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange CreateDnTensorOp::getDimsMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } std::pair CreateDnTensorOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult CreateDnTensorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateDnTensorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateDnTensorOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional CreateDnTensorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void CreateDnTensorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void CreateDnTensorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult CreateDnTensorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult CreateDnTensorOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void CreateDnTensorOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void CreateDnTensorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type dnTensor, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { odsState.addOperands(asyncDependencies); odsState.addOperands(memref); odsState.addOperands(dims); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, static_cast(dims.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(dnTensor); if (asyncToken) odsState.addTypes(asyncToken); } CreateDnTensorOp CreateDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type dnTensor, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dnTensor), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(memref), std::forward(dims)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateDnTensorOp CreateDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type dnTensor, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { return create(builder, builder.getLoc(), std::forward(dnTensor), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(memref), std::forward(dims)); } void CreateDnTensorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { odsState.addOperands(asyncDependencies); odsState.addOperands(memref); odsState.addOperands(dims); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, static_cast(dims.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateDnTensorOp CreateDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(memref), std::forward(dims)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateDnTensorOp CreateDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref, ::mlir::ValueRange dims) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(memref), std::forward(dims)); } void CreateDnTensorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } CreateDnTensorOp CreateDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateDnTensorOp CreateDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateDnTensorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateDnTensorOp CreateDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateDnTensorOp CreateDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateDnTensorOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateDnTensorOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateDnTensorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand memrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> memrefOperands(&memrefRawOperand, 1); ::llvm::SMLoc memrefOperandsLoc; (void)memrefOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> dimsOperands; ::llvm::SMLoc dimsOperandsLoc; (void)dimsOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> dimsTypes; ::mlir::Type memrefRawType{}; ::llvm::ArrayRef<::mlir::Type> memrefTypes(&memrefRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } memrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(memrefRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dimsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(dimsOperands)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(dimsTypes)) return ::mlir::failure(); if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); memrefRawType = type; } ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependenciesOperands.size()), 1, static_cast(dimsOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(memrefOperands, memrefTypes, memrefOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dimsOperands, dimsTypes, dimsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateDnTensorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getMemref(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDims(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getDims().getTypes(); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::CreateDnTensorOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::CreateDnTensorOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::DeallocOp definitions //===----------------------------------------------------------------------===// namespace detail { DeallocOpGenericAdaptorBase::DeallocOpGenericAdaptorBase(DeallocOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DeallocOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail DeallocOpAdaptor::DeallocOpAdaptor(DeallocOp op) : DeallocOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DeallocOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair DeallocOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange DeallocOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair DeallocOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult DeallocOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DeallocOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DeallocOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional DeallocOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void DeallocOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void DeallocOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult DeallocOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void DeallocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(memref); if (asyncToken) odsState.addTypes(asyncToken); } DeallocOp DeallocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeallocOp DeallocOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(memref)); } void DeallocOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { odsState.addOperands(asyncDependencies); odsState.addOperands(memref); odsState.addTypes(resultTypes); } DeallocOp DeallocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(memref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeallocOp DeallocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value memref) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(memref)); } void DeallocOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } DeallocOp DeallocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeallocOp DeallocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DeallocOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } DeallocOp DeallocOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DeallocOp DeallocOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DeallocOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DeallocOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult DeallocOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand memrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> memrefOperands(&memrefRawOperand, 1); ::llvm::SMLoc memrefOperandsLoc; (void)memrefOperandsLoc; ::mlir::Type memrefRawType{}; ::llvm::ArrayRef<::mlir::Type> memrefTypes(&memrefRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } memrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(memrefRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); memrefRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(memrefOperands, memrefTypes, memrefOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DeallocOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getMemref(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void DeallocOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Free::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::DeallocOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::DestroyDnTensorOp definitions //===----------------------------------------------------------------------===// namespace detail { DestroyDnTensorOpGenericAdaptorBase::DestroyDnTensorOpGenericAdaptorBase(DestroyDnTensorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DestroyDnTensorOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail DestroyDnTensorOpAdaptor::DestroyDnTensorOpAdaptor(DestroyDnTensorOp op) : DestroyDnTensorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DestroyDnTensorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair DestroyDnTensorOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange DestroyDnTensorOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair DestroyDnTensorOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult DestroyDnTensorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DestroyDnTensorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DestroyDnTensorOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional DestroyDnTensorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void DestroyDnTensorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void DestroyDnTensorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult DestroyDnTensorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void DestroyDnTensorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnTensor); if (asyncToken) odsState.addTypes(asyncToken); } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnTensor)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnTensor)); } void DestroyDnTensorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnTensor); odsState.addTypes(resultTypes); } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dnTensor)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dnTensor) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dnTensor)); } void DestroyDnTensorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DestroyDnTensorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroyDnTensorOp DestroyDnTensorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DestroyDnTensorOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DestroyDnTensorOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult DestroyDnTensorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dnTensorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnTensorOperands(&dnTensorRawOperand, 1); ::llvm::SMLoc dnTensorOperandsLoc; (void)dnTensorOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } dnTensorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnTensorRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnTensorOperands, odsBuildableType1, dnTensorOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DestroyDnTensorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDnTensor(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::DestroyDnTensorOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::DestroySpMatOp definitions //===----------------------------------------------------------------------===// namespace detail { DestroySpMatOpGenericAdaptorBase::DestroySpMatOpGenericAdaptorBase(DestroySpMatOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DestroySpMatOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail DestroySpMatOpAdaptor::DestroySpMatOpAdaptor(DestroySpMatOp op) : DestroySpMatOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DestroySpMatOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair DestroySpMatOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange DestroySpMatOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair DestroySpMatOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult DestroySpMatOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DestroySpMatOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DestroySpMatOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional DestroySpMatOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void DestroySpMatOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void DestroySpMatOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult DestroySpMatOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void DestroySpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); if (asyncToken) odsState.addTypes(asyncToken); } DestroySpMatOp DestroySpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroySpMatOp DestroySpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat)); } void DestroySpMatOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); odsState.addTypes(resultTypes); } DestroySpMatOp DestroySpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroySpMatOp DestroySpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat)); } void DestroySpMatOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } DestroySpMatOp DestroySpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroySpMatOp DestroySpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DestroySpMatOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } DestroySpMatOp DestroySpMatOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DestroySpMatOp DestroySpMatOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DestroySpMatOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DestroySpMatOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult DestroySpMatOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatOperands(&spmatRawOperand, 1); ::llvm::SMLoc spmatOperandsLoc; (void)spmatOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatOperands, odsBuildableType1, spmatOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DestroySpMatOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmat(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::DestroySpMatOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::DynamicSharedMemoryOp definitions //===----------------------------------------------------------------------===// namespace detail { DynamicSharedMemoryOpGenericAdaptorBase::DynamicSharedMemoryOpGenericAdaptorBase(DynamicSharedMemoryOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail DynamicSharedMemoryOpAdaptor::DynamicSharedMemoryOpAdaptor(DynamicSharedMemoryOp op) : DynamicSharedMemoryOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DynamicSharedMemoryOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult DynamicSharedMemoryOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DynamicSharedMemoryOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DynamicSharedMemoryOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional DynamicSharedMemoryOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void DynamicSharedMemoryOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void DynamicSharedMemoryOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult DynamicSharedMemoryOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void DynamicSharedMemoryOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultMemref) { odsState.addTypes(resultMemref); } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultMemref) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultMemref)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultMemref) { return create(builder, builder.getLoc(), std::forward(resultMemref)); } void DynamicSharedMemoryOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void DynamicSharedMemoryOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DynamicSharedMemoryOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DynamicSharedMemoryOp DynamicSharedMemoryOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DynamicSharedMemoryOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps12(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DynamicSharedMemoryOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult DynamicSharedMemoryOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::Type resultMemrefRawType{}; ::llvm::ArrayRef<::mlir::Type> resultMemrefTypes(&resultMemrefRawType, 1); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultMemrefRawType = type; } result.addTypes(resultMemrefTypes); return ::mlir::success(); } void DynamicSharedMemoryOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResultMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void DynamicSharedMemoryOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::DynamicSharedMemoryOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::GPUFuncOp definitions //===----------------------------------------------------------------------===// namespace detail { GPUFuncOpGenericAdaptorBase::GPUFuncOpGenericAdaptorBase(GPUFuncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::FunctionType GPUFuncOpGenericAdaptorBase::getFunctionType() { auto attr = getFunctionTypeAttr(); return ::llvm::cast<::mlir::FunctionType>(attr.getValue()); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOpGenericAdaptorBase::getArgAttrs() { auto attr = getArgAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOpGenericAdaptorBase::getResAttrs() { auto attr = getResAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOpGenericAdaptorBase::getWorkgroupAttribAttrs() { auto attr = getWorkgroupAttribAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOpGenericAdaptorBase::getPrivateAttribAttrs() { auto attr = getPrivateAttribAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOpGenericAdaptorBase::getKnownBlockSize() { auto attr = getKnownBlockSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOpGenericAdaptorBase::getKnownGridSize() { auto attr = getKnownGridSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOpGenericAdaptorBase::getKnownClusterSize() { auto attr = getKnownClusterSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional GPUFuncOpGenericAdaptorBase::getWorkgroupAttributions() { auto attr = getWorkgroupAttributionsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::mlir::UnitAttr GPUFuncOpGenericAdaptorBase::getKernelAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().kernel); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool GPUFuncOpGenericAdaptorBase::getKernel() { auto attr = getKernelAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } } // namespace detail GPUFuncOpAdaptor::GPUFuncOpAdaptor(GPUFuncOp op) : GPUFuncOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult GPUFuncOpAdaptor::verify(::mlir::Location loc) { auto tblgen_arg_attrs = getProperties().arg_attrs; (void)tblgen_arg_attrs; auto tblgen_function_type = getProperties().function_type; (void)tblgen_function_type; if (!tblgen_function_type) return emitError(loc, "'gpu.func' op requires attribute 'function_type'"); auto tblgen_kernel = getProperties().kernel; (void)tblgen_kernel; auto tblgen_known_block_size = getProperties().known_block_size; (void)tblgen_known_block_size; auto tblgen_known_cluster_size = getProperties().known_cluster_size; (void)tblgen_known_cluster_size; auto tblgen_known_grid_size = getProperties().known_grid_size; (void)tblgen_known_grid_size; auto tblgen_private_attrib_attrs = getProperties().private_attrib_attrs; (void)tblgen_private_attrib_attrs; auto tblgen_res_attrs = getProperties().res_attrs; (void)tblgen_res_attrs; auto tblgen_workgroup_attrib_attrs = getProperties().workgroup_attrib_attrs; (void)tblgen_workgroup_attrib_attrs; auto tblgen_workgroup_attributions = getProperties().workgroup_attributions; (void)tblgen_workgroup_attributions; if (tblgen_function_type && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_function_type))) && ((::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>(tblgen_function_type).getValue()))) && ((::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>(tblgen_function_type).getValue()))))) return emitError(loc, "'gpu.func' op attribute 'function_type' failed to satisfy constraint: type attribute of function type"); if (tblgen_arg_attrs && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_arg_attrs))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_arg_attrs), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::DictionaryAttr>(attr))); })))) return emitError(loc, "'gpu.func' op attribute 'arg_attrs' failed to satisfy constraint: Array of dictionary attributes"); if (tblgen_res_attrs && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_res_attrs))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_res_attrs), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::DictionaryAttr>(attr))); })))) return emitError(loc, "'gpu.func' op attribute 'res_attrs' failed to satisfy constraint: Array of dictionary attributes"); if (tblgen_workgroup_attrib_attrs && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_workgroup_attrib_attrs))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_workgroup_attrib_attrs), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::DictionaryAttr>(attr))); })))) return emitError(loc, "'gpu.func' op attribute 'workgroup_attrib_attrs' failed to satisfy constraint: Array of dictionary attributes"); if (tblgen_private_attrib_attrs && !(((::llvm::isa<::mlir::ArrayAttr>(tblgen_private_attrib_attrs))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_private_attrib_attrs), [&](::mlir::Attribute attr) { return attr && ((::llvm::isa<::mlir::DictionaryAttr>(attr))); })))) return emitError(loc, "'gpu.func' op attribute 'private_attrib_attrs' failed to satisfy constraint: Array of dictionary attributes"); if (tblgen_known_block_size && !(((::llvm::isa<::mlir::DenseI32ArrayAttr>(tblgen_known_block_size))) && (((!tblgen_known_block_size)) || (((::llvm::cast<::mlir::DenseArrayAttr>(tblgen_known_block_size).size() == 3)) && ((([](mlir::Attribute a) { auto arr = ::llvm::cast<::mlir::DenseI32ArrayAttr>(a).asArrayRef(); return ::llvm::all_of(arr, [](int32_t v) { return v >= 1; });})(tblgen_known_block_size))))))) return emitError(loc, "'gpu.func' op attribute 'known_block_size' failed to satisfy constraint: i32 dense array attribute with 3 elements (if present) and all elements >= 1"); if (tblgen_known_grid_size && !(((::llvm::isa<::mlir::DenseI32ArrayAttr>(tblgen_known_grid_size))) && (((!tblgen_known_grid_size)) || (((::llvm::cast<::mlir::DenseArrayAttr>(tblgen_known_grid_size).size() == 3)) && ((([](mlir::Attribute a) { auto arr = ::llvm::cast<::mlir::DenseI32ArrayAttr>(a).asArrayRef(); return ::llvm::all_of(arr, [](int32_t v) { return v >= 1; });})(tblgen_known_grid_size))))))) return emitError(loc, "'gpu.func' op attribute 'known_grid_size' failed to satisfy constraint: i32 dense array attribute with 3 elements (if present) and all elements >= 1"); if (tblgen_known_cluster_size && !(((::llvm::isa<::mlir::DenseI32ArrayAttr>(tblgen_known_cluster_size))) && (((!tblgen_known_cluster_size)) || (((::llvm::cast<::mlir::DenseArrayAttr>(tblgen_known_cluster_size).size() == 3)) && ((([](mlir::Attribute a) { auto arr = ::llvm::cast<::mlir::DenseI32ArrayAttr>(a).asArrayRef(); return ::llvm::all_of(arr, [](int32_t v) { return v >= 1; });})(tblgen_known_cluster_size))))))) return emitError(loc, "'gpu.func' op attribute 'known_cluster_size' failed to satisfy constraint: i32 dense array attribute with 3 elements (if present) and all elements >= 1"); if (tblgen_workgroup_attributions && !((((::llvm::isa<::mlir::IntegerAttr>(tblgen_workgroup_attributions))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_workgroup_attributions).getType().isSignlessInteger(64)))) && ((!::llvm::cast<::mlir::IntegerAttr>(tblgen_workgroup_attributions).getValue().isNegative())))) return emitError(loc, "'gpu.func' op attribute 'workgroup_attributions' failed to satisfy constraint: 64-bit signless integer attribute whose value is non-negative"); if (tblgen_kernel && !((::llvm::isa<::mlir::UnitAttr>(tblgen_kernel)))) return emitError(loc, "'gpu.func' op attribute 'kernel' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult GPUFuncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.arg_attrs; auto attr = dict.get("arg_attrs"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `arg_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.function_type; auto attr = dict.get("function_type"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `function_type` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.kernel; auto attr = dict.get("kernel"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kernel` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_block_size; auto attr = dict.get("known_block_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_block_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_cluster_size; auto attr = dict.get("known_cluster_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_cluster_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_grid_size; auto attr = dict.get("known_grid_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_grid_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.private_attrib_attrs; auto attr = dict.get("private_attrib_attrs"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `private_attrib_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.res_attrs; auto attr = dict.get("res_attrs"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `res_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attrib_attrs; auto attr = dict.get("workgroup_attrib_attrs"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attrib_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attributions; auto attr = dict.get("workgroup_attributions"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attributions` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute GPUFuncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.arg_attrs; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("arg_attrs", propStorage)); } { const auto &propStorage = prop.function_type; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("function_type", propStorage)); } { const auto &propStorage = prop.kernel; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("kernel", propStorage)); } { const auto &propStorage = prop.known_block_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("known_block_size", propStorage)); } { const auto &propStorage = prop.known_cluster_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("known_cluster_size", propStorage)); } { const auto &propStorage = prop.known_grid_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("known_grid_size", propStorage)); } { const auto &propStorage = prop.private_attrib_attrs; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("private_attrib_attrs", propStorage)); } { const auto &propStorage = prop.res_attrs; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("res_attrs", propStorage)); } { const auto &propStorage = prop.workgroup_attrib_attrs; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("workgroup_attrib_attrs", propStorage)); } { const auto &propStorage = prop.workgroup_attributions; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("workgroup_attributions", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code GPUFuncOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.arg_attrs.getAsOpaquePointer()), llvm::hash_value(prop.function_type.getAsOpaquePointer()), llvm::hash_value(prop.kernel.getAsOpaquePointer()), llvm::hash_value(prop.known_block_size.getAsOpaquePointer()), llvm::hash_value(prop.known_cluster_size.getAsOpaquePointer()), llvm::hash_value(prop.known_grid_size.getAsOpaquePointer()), llvm::hash_value(prop.private_attrib_attrs.getAsOpaquePointer()), llvm::hash_value(prop.res_attrs.getAsOpaquePointer()), llvm::hash_value(prop.workgroup_attrib_attrs.getAsOpaquePointer()), llvm::hash_value(prop.workgroup_attributions.getAsOpaquePointer())); } std::optional GPUFuncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "arg_attrs") return prop.arg_attrs; if (name == "function_type") return prop.function_type; if (name == "kernel") return prop.kernel; if (name == "known_block_size") return prop.known_block_size; if (name == "known_cluster_size") return prop.known_cluster_size; if (name == "known_grid_size") return prop.known_grid_size; if (name == "private_attrib_attrs") return prop.private_attrib_attrs; if (name == "res_attrs") return prop.res_attrs; if (name == "workgroup_attrib_attrs") return prop.workgroup_attrib_attrs; if (name == "workgroup_attributions") return prop.workgroup_attributions; return std::nullopt; } void GPUFuncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "arg_attrs") { prop.arg_attrs = ::llvm::dyn_cast_or_null>(value); return; } if (name == "function_type") { prop.function_type = ::llvm::dyn_cast_or_null>(value); return; } if (name == "kernel") { prop.kernel = ::llvm::dyn_cast_or_null>(value); return; } if (name == "known_block_size") { prop.known_block_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "known_cluster_size") { prop.known_cluster_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "known_grid_size") { prop.known_grid_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "private_attrib_attrs") { prop.private_attrib_attrs = ::llvm::dyn_cast_or_null>(value); return; } if (name == "res_attrs") { prop.res_attrs = ::llvm::dyn_cast_or_null>(value); return; } if (name == "workgroup_attrib_attrs") { prop.workgroup_attrib_attrs = ::llvm::dyn_cast_or_null>(value); return; } if (name == "workgroup_attributions") { prop.workgroup_attributions = ::llvm::dyn_cast_or_null>(value); return; } } void GPUFuncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.arg_attrs) attrs.append("arg_attrs", prop.arg_attrs); if (prop.function_type) attrs.append("function_type", prop.function_type); if (prop.kernel) attrs.append("kernel", prop.kernel); if (prop.known_block_size) attrs.append("known_block_size", prop.known_block_size); if (prop.known_cluster_size) attrs.append("known_cluster_size", prop.known_cluster_size); if (prop.known_grid_size) attrs.append("known_grid_size", prop.known_grid_size); if (prop.private_attrib_attrs) attrs.append("private_attrib_attrs", prop.private_attrib_attrs); if (prop.res_attrs) attrs.append("res_attrs", prop.res_attrs); if (prop.workgroup_attrib_attrs) attrs.append("workgroup_attrib_attrs", prop.workgroup_attrib_attrs); if (prop.workgroup_attributions) attrs.append("workgroup_attributions", prop.workgroup_attributions); } ::llvm::LogicalResult GPUFuncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getArgAttrsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(attr, "arg_attrs", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getFunctionTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps11(attr, "function_type", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKernelAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "kernel", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKnownBlockSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(attr, "known_block_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKnownClusterSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(attr, "known_cluster_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKnownGridSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(attr, "known_grid_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getPrivateAttribAttrsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(attr, "private_attrib_attrs", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getResAttrsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(attr, "res_attrs", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getWorkgroupAttribAttrsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(attr, "workgroup_attrib_attrs", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getWorkgroupAttributionsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps14(attr, "workgroup_attributions", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GPUFuncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.arg_attrs))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.function_type))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.kernel))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.known_block_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.known_cluster_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.known_grid_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.private_attrib_attrs))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.res_attrs))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.workgroup_attrib_attrs))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.workgroup_attributions))) return ::mlir::failure(); return ::mlir::success(); } void GPUFuncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.arg_attrs); writer.writeAttribute(prop.function_type); writer.writeOptionalAttribute(prop.kernel); writer.writeOptionalAttribute(prop.known_block_size); writer.writeOptionalAttribute(prop.known_cluster_size); writer.writeOptionalAttribute(prop.known_grid_size); writer.writeOptionalAttribute(prop.private_attrib_attrs); writer.writeOptionalAttribute(prop.res_attrs); writer.writeOptionalAttribute(prop.workgroup_attrib_attrs); writer.writeOptionalAttribute(prop.workgroup_attributions); } ::mlir::FunctionType GPUFuncOp::getFunctionType() { auto attr = getFunctionTypeAttr(); return ::llvm::cast<::mlir::FunctionType>(attr.getValue()); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOp::getArgAttrs() { auto attr = getArgAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOp::getResAttrs() { auto attr = getResAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOp::getWorkgroupAttribAttrs() { auto attr = getWorkgroupAttribAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional< ::mlir::ArrayAttr > GPUFuncOp::getPrivateAttribAttrs() { auto attr = getPrivateAttribAttrsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOp::getKnownBlockSize() { auto attr = getKnownBlockSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOp::getKnownGridSize() { auto attr = getKnownGridSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> GPUFuncOp::getKnownClusterSize() { auto attr = getKnownClusterSizeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional GPUFuncOp::getWorkgroupAttributions() { auto attr = getWorkgroupAttributionsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } bool GPUFuncOp::getKernel() { auto attr = getKernelAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } void GPUFuncOp::setFunctionType(::mlir::FunctionType attrValue) { getProperties().function_type = ::mlir::TypeAttr::get(attrValue); } void GPUFuncOp::setKnownBlockSize(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().known_block_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI32ArrayAttr(*attrValue); else odsProp = nullptr; } void GPUFuncOp::setKnownGridSize(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().known_grid_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI32ArrayAttr(*attrValue); else odsProp = nullptr; } void GPUFuncOp::setKnownClusterSize(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().known_cluster_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI32ArrayAttr(*attrValue); else odsProp = nullptr; } void GPUFuncOp::setWorkgroupAttributions(::std::optional attrValue) { auto &odsProp = getProperties().workgroup_attributions; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } void GPUFuncOp::setKernel(bool attrValue) { auto &odsProp = getProperties().kernel; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } GPUFuncOp GPUFuncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, StringRef name, FunctionType type, TypeRange workgroupAttributions, TypeRange privateAttributions, ArrayRef attrs) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(name), std::forward(type), std::forward(workgroupAttributions), std::forward(privateAttributions), std::forward(attrs)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GPUFuncOp GPUFuncOp::create(::mlir::ImplicitLocOpBuilder &builder, StringRef name, FunctionType type, TypeRange workgroupAttributions, TypeRange privateAttributions, ArrayRef attrs) { return create(builder, builder.getLoc(), std::forward(name), std::forward(type), std::forward(workgroupAttributions), std::forward(privateAttributions), std::forward(attrs)); } ::llvm::LogicalResult GPUFuncOp::verifyInvariantsImpl() { auto tblgen_arg_attrs = getProperties().arg_attrs; (void)tblgen_arg_attrs; auto tblgen_function_type = getProperties().function_type; (void)tblgen_function_type; if (!tblgen_function_type) return emitOpError("requires attribute 'function_type'"); auto tblgen_kernel = getProperties().kernel; (void)tblgen_kernel; auto tblgen_known_block_size = getProperties().known_block_size; (void)tblgen_known_block_size; auto tblgen_known_cluster_size = getProperties().known_cluster_size; (void)tblgen_known_cluster_size; auto tblgen_known_grid_size = getProperties().known_grid_size; (void)tblgen_known_grid_size; auto tblgen_private_attrib_attrs = getProperties().private_attrib_attrs; (void)tblgen_private_attrib_attrs; auto tblgen_res_attrs = getProperties().res_attrs; (void)tblgen_res_attrs; auto tblgen_workgroup_attrib_attrs = getProperties().workgroup_attrib_attrs; (void)tblgen_workgroup_attrib_attrs; auto tblgen_workgroup_attributions = getProperties().workgroup_attributions; (void)tblgen_workgroup_attributions; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps11(*this, tblgen_function_type, "function_type"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(*this, tblgen_arg_attrs, "arg_attrs"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(*this, tblgen_res_attrs, "res_attrs"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(*this, tblgen_workgroup_attrib_attrs, "workgroup_attrib_attrs"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps12(*this, tblgen_private_attrib_attrs, "private_attrib_attrs"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(*this, tblgen_known_block_size, "known_block_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(*this, tblgen_known_grid_size, "known_grid_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps13(*this, tblgen_known_cluster_size, "known_cluster_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps14(*this, tblgen_workgroup_attributions, "workgroup_attributions"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_kernel, "kernel"))) return ::mlir::failure(); { unsigned index = 0; (void)index; for (auto ®ion : ::llvm::MutableArrayRef((*this)->getRegion(0))) if (::mlir::failed(__mlir_ods_local_region_constraint_GPUOps1(*this, region, "body", index++))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GPUFuncOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult GPUFuncOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.function_type; auto function_typeAttrName = ::mlir::StringAttr::get(ctx, "function_type"); auto attr = dict.get(function_typeAttrName); usedKeys.insert(function_typeAttrName); if (attr || /*isRequired=*/true) { if (!attr) { emitError() << "expected key entry for function_type in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `function_type` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.arg_attrs; auto arg_attrsAttrName = ::mlir::StringAttr::get(ctx, "arg_attrs"); auto attr = dict.get(arg_attrsAttrName); usedKeys.insert(arg_attrsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for arg_attrs in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `arg_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.res_attrs; auto res_attrsAttrName = ::mlir::StringAttr::get(ctx, "res_attrs"); auto attr = dict.get(res_attrsAttrName); usedKeys.insert(res_attrsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for res_attrs in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `res_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attrib_attrs; auto workgroup_attrib_attrsAttrName = ::mlir::StringAttr::get(ctx, "workgroup_attrib_attrs"); auto attr = dict.get(workgroup_attrib_attrsAttrName); usedKeys.insert(workgroup_attrib_attrsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for workgroup_attrib_attrs in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attrib_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.private_attrib_attrs; auto private_attrib_attrsAttrName = ::mlir::StringAttr::get(ctx, "private_attrib_attrs"); auto attr = dict.get(private_attrib_attrsAttrName); usedKeys.insert(private_attrib_attrsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for private_attrib_attrs in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `private_attrib_attrs` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_block_size; auto known_block_sizeAttrName = ::mlir::StringAttr::get(ctx, "known_block_size"); auto attr = dict.get(known_block_sizeAttrName); usedKeys.insert(known_block_sizeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for known_block_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_block_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_grid_size; auto known_grid_sizeAttrName = ::mlir::StringAttr::get(ctx, "known_grid_size"); auto attr = dict.get(known_grid_sizeAttrName); usedKeys.insert(known_grid_sizeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for known_grid_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_grid_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.known_cluster_size; auto known_cluster_sizeAttrName = ::mlir::StringAttr::get(ctx, "known_cluster_size"); auto attr = dict.get(known_cluster_sizeAttrName); usedKeys.insert(known_cluster_sizeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for known_cluster_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `known_cluster_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attributions; auto workgroup_attributionsAttrName = ::mlir::StringAttr::get(ctx, "workgroup_attributions"); auto attr = dict.get(workgroup_attributionsAttrName); usedKeys.insert(workgroup_attributionsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for workgroup_attributions in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attributions` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.kernel; auto kernelAttrName = ::mlir::StringAttr::get(ctx, "kernel"); auto attr = dict.get(kernelAttrName); usedKeys.insert(kernelAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for kernel in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kernel` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::GPUFuncOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::GPUFuncOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::GPUModuleOp definitions //===----------------------------------------------------------------------===// namespace detail { GPUModuleOpGenericAdaptorBase::GPUModuleOpGenericAdaptorBase(GPUModuleOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::llvm::StringRef GPUModuleOpGenericAdaptorBase::getSymName() { auto attr = getSymNameAttr(); return attr.getValue(); } ::std::optional< ::mlir::ArrayAttr > GPUModuleOpGenericAdaptorBase::getTargets() { auto attr = getTargetsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional<::mlir::Attribute> GPUModuleOpGenericAdaptorBase::getOffloadingHandler() { auto attr = getOffloadingHandlerAttr(); return attr ? ::std::optional<::mlir::Attribute>(attr) : (::std::nullopt); } } // namespace detail GPUModuleOpAdaptor::GPUModuleOpAdaptor(GPUModuleOp op) : GPUModuleOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult GPUModuleOpAdaptor::verify(::mlir::Location loc) { auto tblgen_offloadingHandler = getProperties().offloadingHandler; (void)tblgen_offloadingHandler; auto tblgen_sym_name = getProperties().sym_name; (void)tblgen_sym_name; if (!tblgen_sym_name) return emitError(loc, "'gpu.module' op requires attribute 'sym_name'"); auto tblgen_targets = getProperties().targets; (void)tblgen_targets; if (tblgen_sym_name && !((::llvm::isa<::mlir::StringAttr>(tblgen_sym_name)))) return emitError(loc, "'gpu.module' op attribute 'sym_name' failed to satisfy constraint: string attribute"); if (tblgen_targets && !((((::llvm::isa<::mlir::ArrayAttr>(tblgen_targets))) && (::llvm::all_of(::llvm::cast<::mlir::ArrayAttr>(tblgen_targets), [&](::mlir::Attribute attr) { return attr && (((true)) && ((attr.hasPromiseOrImplementsInterface<::mlir::gpu::TargetAttrInterface>()))); }))) && ((::llvm::cast<::mlir::ArrayAttr>(tblgen_targets).size() >= 1)))) return emitError(loc, "'gpu.module' op attribute 'targets' failed to satisfy constraint: array of GPU target attributes with at least 1 elements"); if (tblgen_offloadingHandler && !(((true)) && ((tblgen_offloadingHandler.hasTrait<::mlir::gpu::OffloadingTranslationAttrTrait>())))) return emitError(loc, "'gpu.module' op attribute 'offloadingHandler' failed to satisfy constraint: any attribute with the `OffloadingTranslationAttrTrait` trait."); return ::mlir::success(); } ::llvm::LogicalResult GPUModuleOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.offloadingHandler; auto attr = dict.get("offloadingHandler"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `offloadingHandler` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.sym_name; auto attr = dict.get("sym_name"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `sym_name` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.targets; auto attr = dict.get("targets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `targets` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute GPUModuleOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.offloadingHandler; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("offloadingHandler", propStorage)); } { const auto &propStorage = prop.sym_name; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("sym_name", propStorage)); } { const auto &propStorage = prop.targets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("targets", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code GPUModuleOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.offloadingHandler.getAsOpaquePointer()), llvm::hash_value(prop.sym_name.getAsOpaquePointer()), llvm::hash_value(prop.targets.getAsOpaquePointer())); } std::optional GPUModuleOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "offloadingHandler") return prop.offloadingHandler; if (name == "sym_name") return prop.sym_name; if (name == "targets") return prop.targets; return std::nullopt; } void GPUModuleOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "offloadingHandler") { prop.offloadingHandler = ::llvm::dyn_cast_or_null>(value); return; } if (name == "sym_name") { prop.sym_name = ::llvm::dyn_cast_or_null>(value); return; } if (name == "targets") { prop.targets = ::llvm::dyn_cast_or_null>(value); return; } } void GPUModuleOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.offloadingHandler) attrs.append("offloadingHandler", prop.offloadingHandler); if (prop.sym_name) attrs.append("sym_name", prop.sym_name); if (prop.targets) attrs.append("targets", prop.targets); } ::llvm::LogicalResult GPUModuleOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getOffloadingHandlerAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps6(attr, "offloadingHandler", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getSymNameAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(attr, "sym_name", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTargetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps15(attr, "targets", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GPUModuleOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.offloadingHandler))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.sym_name))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.targets))) return ::mlir::failure(); return ::mlir::success(); } void GPUModuleOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.offloadingHandler); writer.writeAttribute(prop.sym_name); writer.writeOptionalAttribute(prop.targets); } ::llvm::StringRef GPUModuleOp::getSymName() { auto attr = getSymNameAttr(); return attr.getValue(); } ::std::optional< ::mlir::ArrayAttr > GPUModuleOp::getTargets() { auto attr = getTargetsAttr(); return attr ? ::std::optional< ::mlir::ArrayAttr >(attr) : (::std::nullopt); } ::std::optional<::mlir::Attribute> GPUModuleOp::getOffloadingHandler() { auto attr = getOffloadingHandlerAttr(); return attr ? ::std::optional<::mlir::Attribute>(attr) : (::std::nullopt); } void GPUModuleOp::setSymName(::llvm::StringRef attrValue) { getProperties().sym_name = ::mlir::Builder((*this)->getContext()).getStringAttr(attrValue); } GPUModuleOp GPUModuleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, StringRef name, ArrayAttr targets, Attribute handler) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(name), std::forward(targets), std::forward(handler)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GPUModuleOp GPUModuleOp::create(::mlir::ImplicitLocOpBuilder &builder, StringRef name, ArrayAttr targets, Attribute handler) { return create(builder, builder.getLoc(), std::forward(name), std::forward(targets), std::forward(handler)); } GPUModuleOp GPUModuleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, StringRef name, ArrayRef targets, Attribute handler) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(name), std::forward(targets), std::forward(handler)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GPUModuleOp GPUModuleOp::create(::mlir::ImplicitLocOpBuilder &builder, StringRef name, ArrayRef targets, Attribute handler) { return create(builder, builder.getLoc(), std::forward(name), std::forward(targets), std::forward(handler)); } ::llvm::LogicalResult GPUModuleOp::verifyInvariantsImpl() { auto tblgen_offloadingHandler = getProperties().offloadingHandler; (void)tblgen_offloadingHandler; auto tblgen_sym_name = getProperties().sym_name; (void)tblgen_sym_name; if (!tblgen_sym_name) return emitOpError("requires attribute 'sym_name'"); auto tblgen_targets = getProperties().targets; (void)tblgen_targets; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(*this, tblgen_sym_name, "sym_name"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps15(*this, tblgen_targets, "targets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps6(*this, tblgen_offloadingHandler, "offloadingHandler"))) return ::mlir::failure(); { unsigned index = 0; (void)index; for (auto ®ion : ::llvm::MutableArrayRef((*this)->getRegion(0))) if (::mlir::failed(__mlir_ods_local_region_constraint_GPUOps2(*this, region, "bodyRegion", index++))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GPUModuleOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult GPUModuleOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::StringAttr sym_nameAttr; ::mlir::Attribute offloadingHandlerAttr; ::mlir::ArrayAttr targetsAttr; std::unique_ptr<::mlir::Region> bodyRegionRegion = std::make_unique<::mlir::Region>(); if (parser.parseSymbolName(sym_nameAttr)) return ::mlir::failure(); if (sym_nameAttr) result.getOrAddProperties().sym_name = sym_nameAttr; if (::mlir::succeeded(parser.parseOptionalLess())) { if (parser.parseAttribute(offloadingHandlerAttr, ::mlir::Type{})) return ::mlir::failure(); if (offloadingHandlerAttr) result.getOrAddProperties().offloadingHandler = offloadingHandlerAttr; if (parser.parseGreater()) return ::mlir::failure(); } ::mlir::OptionalParseResult parseResulttargetsAttr = parser.parseOptionalAttribute(targetsAttr, parser.getBuilder().getType<::mlir::NoneType>()); if (parseResulttargetsAttr.has_value() && failed(*parseResulttargetsAttr)) return ::mlir::failure(); if (parseResulttargetsAttr.has_value() && succeeded(*parseResulttargetsAttr)) if (targetsAttr) result.getOrAddProperties().targets = targetsAttr; if (targetsAttr) { } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDictWithKeyword(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseRegion(*bodyRegionRegion)) return ::mlir::failure(); if (bodyRegionRegion->empty()) bodyRegionRegion->emplaceBlock(); result.addRegion(std::move(bodyRegionRegion)); return ::mlir::success(); } void GPUModuleOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printSymbolName(getSymNameAttr().getValue()); if (getOffloadingHandlerAttr()) { _odsPrinter << "<"; _odsPrinter.printAttribute(getOffloadingHandlerAttr()); _odsPrinter << ">"; } if (getTargetsAttr()) { _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getTargetsAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("sym_name"); elidedAttrs.push_back("offloadingHandler"); elidedAttrs.push_back("targets"); _odsPrinter.printOptionalAttrDictWithKeyword((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' '; _odsPrinter.printRegion(getBodyRegion()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::GPUModuleOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::GPUModuleOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::GlobalIdOp definitions //===----------------------------------------------------------------------===// namespace detail { GlobalIdOpGenericAdaptorBase::GlobalIdOpGenericAdaptorBase(GlobalIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension GlobalIdOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > GlobalIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail GlobalIdOpAdaptor::GlobalIdOpAdaptor(GlobalIdOp op) : GlobalIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult GlobalIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.global_id' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.global_id' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.global_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult GlobalIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute GlobalIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code GlobalIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional GlobalIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void GlobalIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void GlobalIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult GlobalIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GlobalIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void GlobalIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension GlobalIdOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > GlobalIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void GlobalIdOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void GlobalIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GlobalIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GlobalIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void GlobalIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GlobalIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void GlobalIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void GlobalIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GlobalIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } GlobalIdOp GlobalIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GlobalIdOp GlobalIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult GlobalIdOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult GlobalIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult GlobalIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult GlobalIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void GlobalIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void GlobalIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void GlobalIdOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::GlobalIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::GlobalIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::GridDimOp definitions //===----------------------------------------------------------------------===// namespace detail { GridDimOpGenericAdaptorBase::GridDimOpGenericAdaptorBase(GridDimOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension GridDimOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > GridDimOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail GridDimOpAdaptor::GridDimOpAdaptor(GridDimOp op) : GridDimOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult GridDimOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.grid_dim' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.grid_dim' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.grid_dim' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult GridDimOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute GridDimOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code GridDimOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional GridDimOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void GridDimOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void GridDimOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult GridDimOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult GridDimOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void GridDimOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension GridDimOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > GridDimOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void GridDimOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void GridDimOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GridDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GridDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void GridDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GridDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void GridDimOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void GridDimOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(GridDimOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } GridDimOp GridDimOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GridDimOp GridDimOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult GridDimOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult GridDimOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult GridDimOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult GridDimOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void GridDimOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void GridDimOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void GridDimOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::GridDimOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::GridDimOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::HostRegisterOp definitions //===----------------------------------------------------------------------===// namespace detail { HostRegisterOpGenericAdaptorBase::HostRegisterOpGenericAdaptorBase(HostRegisterOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail HostRegisterOpAdaptor::HostRegisterOpAdaptor(HostRegisterOp op) : HostRegisterOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult HostRegisterOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult HostRegisterOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute HostRegisterOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code HostRegisterOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional HostRegisterOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void HostRegisterOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void HostRegisterOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult HostRegisterOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void HostRegisterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value) { odsState.addOperands(value); } HostRegisterOp HostRegisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostRegisterOp HostRegisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(value)); } void HostRegisterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value) { odsState.addOperands(value); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } HostRegisterOp HostRegisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostRegisterOp HostRegisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value)); } void HostRegisterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } HostRegisterOp HostRegisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostRegisterOp HostRegisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void HostRegisterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } HostRegisterOp HostRegisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostRegisterOp HostRegisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult HostRegisterOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult HostRegisterOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult HostRegisterOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::UnrankedMemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void HostRegisterOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::UnrankedMemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::HostRegisterOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::HostUnregisterOp definitions //===----------------------------------------------------------------------===// namespace detail { HostUnregisterOpGenericAdaptorBase::HostUnregisterOpGenericAdaptorBase(HostUnregisterOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail HostUnregisterOpAdaptor::HostUnregisterOpAdaptor(HostUnregisterOp op) : HostUnregisterOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult HostUnregisterOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult HostUnregisterOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute HostUnregisterOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code HostUnregisterOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional HostUnregisterOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void HostUnregisterOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void HostUnregisterOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult HostUnregisterOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void HostUnregisterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value) { odsState.addOperands(value); } HostUnregisterOp HostUnregisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostUnregisterOp HostUnregisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(value)); } void HostUnregisterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value) { odsState.addOperands(value); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } HostUnregisterOp HostUnregisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostUnregisterOp HostUnregisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value)); } void HostUnregisterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } HostUnregisterOp HostUnregisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostUnregisterOp HostUnregisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void HostUnregisterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } HostUnregisterOp HostUnregisterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } HostUnregisterOp HostUnregisterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult HostUnregisterOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult HostUnregisterOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult HostUnregisterOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::UnrankedMemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void HostUnregisterOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::UnrankedMemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::HostUnregisterOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::InitializeNamedBarrierOp definitions //===----------------------------------------------------------------------===// namespace detail { InitializeNamedBarrierOpGenericAdaptorBase::InitializeNamedBarrierOpGenericAdaptorBase(InitializeNamedBarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail InitializeNamedBarrierOpAdaptor::InitializeNamedBarrierOpAdaptor(InitializeNamedBarrierOp op) : InitializeNamedBarrierOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult InitializeNamedBarrierOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult InitializeNamedBarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute InitializeNamedBarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code InitializeNamedBarrierOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional InitializeNamedBarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void InitializeNamedBarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void InitializeNamedBarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult InitializeNamedBarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value member_count) { odsState.addOperands(member_count); odsState.addTypes(result); } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value member_count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(member_count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value member_count) { return create(builder, builder.getLoc(), std::forward(result), std::forward(member_count)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value member_count) { odsState.addOperands(member_count); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InitializeNamedBarrierOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value member_count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(member_count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value member_count) { return create(builder, builder.getLoc(), std::forward(member_count)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value member_count) { odsState.addOperands(member_count); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value member_count) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(member_count)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value member_count) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(member_count)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InitializeNamedBarrierOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void InitializeNamedBarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InitializeNamedBarrierOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitializeNamedBarrierOp InitializeNamedBarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult InitializeNamedBarrierOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps8(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult InitializeNamedBarrierOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult InitializeNamedBarrierOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = mlir::gpu::NamedBarrierType::get(odsBuilder.getContext()); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult InitializeNamedBarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand member_countRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> member_countOperands(&member_countRawOperand, 1); ::llvm::SMLoc member_countOperandsLoc; (void)member_countOperandsLoc; ::mlir::Type member_countRawType{}; ::llvm::ArrayRef<::mlir::Type> member_countTypes(&member_countRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); member_countOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(member_countRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IntegerType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); member_countRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); if (parser.resolveOperands(member_countOperands, member_countTypes, member_countOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void InitializeNamedBarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getMemberCount(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMemberCount().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void InitializeNamedBarrierOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Allocate::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::InitializeNamedBarrierOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::LaneIdOp definitions //===----------------------------------------------------------------------===// namespace detail { LaneIdOpGenericAdaptorBase::LaneIdOpGenericAdaptorBase(LaneIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional< ::llvm::APInt > LaneIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail LaneIdOpAdaptor::LaneIdOpAdaptor(LaneIdOp op) : LaneIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LaneIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.lane_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult LaneIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LaneIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LaneIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional LaneIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void LaneIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void LaneIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult LaneIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LaneIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void LaneIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.upper_bound); } ::std::optional< ::llvm::APInt > LaneIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void LaneIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void LaneIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(result); } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(result), std::forward(upper_bound)); } void LaneIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LaneIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(upper_bound)); } void LaneIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(upper_bound)); } void LaneIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LaneIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LaneIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void LaneIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void LaneIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LaneIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } LaneIdOp LaneIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaneIdOp LaneIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult LaneIdOp::verifyInvariantsImpl() { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult LaneIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult LaneIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult LaneIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::IntegerAttr upper_boundAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void LaneIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void LaneIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::LaneIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::LaneIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::LaunchFuncOp definitions //===----------------------------------------------------------------------===// namespace detail { LaunchFuncOpGenericAdaptorBase::LaunchFuncOpGenericAdaptorBase(LaunchFuncOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LaunchFuncOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::SymbolRefAttr LaunchFuncOpGenericAdaptorBase::getKernel() { auto attr = getKernelAttr(); return attr; } ::mlir::UnitAttr LaunchFuncOpGenericAdaptorBase::getCooperativeAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().cooperative); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool LaunchFuncOpGenericAdaptorBase::getCooperative() { auto attr = getCooperativeAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } } // namespace detail LaunchFuncOpAdaptor::LaunchFuncOpAdaptor(LaunchFuncOp op) : LaunchFuncOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LaunchFuncOpAdaptor::verify(::mlir::Location loc) { auto tblgen_cooperative = getProperties().cooperative; (void)tblgen_cooperative; auto tblgen_kernel = getProperties().kernel; (void)tblgen_kernel; if (!tblgen_kernel) return emitError(loc, "'gpu.launch_func' op requires attribute 'kernel'"); if (tblgen_kernel && !((::llvm::isa<::mlir::SymbolRefAttr>(tblgen_kernel)))) return emitError(loc, "'gpu.launch_func' op attribute 'kernel' failed to satisfy constraint: symbol reference attribute"); if (tblgen_cooperative && !((::llvm::isa<::mlir::UnitAttr>(tblgen_cooperative)))) return emitError(loc, "'gpu.launch_func' op attribute 'cooperative' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } std::pair LaunchFuncOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange LaunchFuncOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(0u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getClusterSizeXMutable() { auto range = getODSOperandIndexAndLength(7); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(7u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getClusterSizeYMutable() { auto range = getODSOperandIndexAndLength(8); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(8u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getClusterSizeZMutable() { auto range = getODSOperandIndexAndLength(9); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(9u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getDynamicSharedMemorySizeMutable() { auto range = getODSOperandIndexAndLength(10); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(10u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getKernelOperandsMutable() { auto range = getODSOperandIndexAndLength(11); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(11u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchFuncOp::getAsyncObjectMutable() { auto range = getODSOperandIndexAndLength(12); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(12u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } std::pair LaunchFuncOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult LaunchFuncOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.cooperative; auto attr = dict.get("cooperative"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `cooperative` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.kernel; auto attr = dict.get("kernel"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kernel` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute LaunchFuncOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.cooperative; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("cooperative", propStorage)); } { const auto &propStorage = prop.kernel; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("kernel", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LaunchFuncOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.cooperative.getAsOpaquePointer()), llvm::hash_value(prop.kernel.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional LaunchFuncOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "cooperative") return prop.cooperative; if (name == "kernel") return prop.kernel; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void LaunchFuncOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "cooperative") { prop.cooperative = ::llvm::dyn_cast_or_null>(value); return; } if (name == "kernel") { prop.kernel = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void LaunchFuncOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.cooperative) attrs.append("cooperative", prop.cooperative); if (prop.kernel) attrs.append("kernel", prop.kernel); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult LaunchFuncOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getCooperativeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "cooperative", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKernelAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps16(attr, "kernel", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LaunchFuncOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.cooperative))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.kernel))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void LaunchFuncOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.cooperative); writer.writeAttribute(prop.kernel); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::SymbolRefAttr LaunchFuncOp::getKernel() { auto attr = getKernelAttr(); return attr; } bool LaunchFuncOp::getCooperative() { auto attr = getCooperativeAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } void LaunchFuncOp::setCooperative(bool attrValue) { auto &odsProp = getProperties().cooperative; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } LaunchFuncOp LaunchFuncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, GPUFuncOp kernelFunc, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Type asyncTokenType, ValueRange asyncDependencies, std::optional clusterSize) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(kernelFunc), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(clusterSize)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaunchFuncOp LaunchFuncOp::create(::mlir::ImplicitLocOpBuilder &builder, GPUFuncOp kernelFunc, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Type asyncTokenType, ValueRange asyncDependencies, std::optional clusterSize) { return create(builder, builder.getLoc(), std::forward(kernelFunc), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(clusterSize)); } LaunchFuncOp LaunchFuncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, SymbolRefAttr kernel, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Type asyncTokenType, ValueRange asyncDependencies, std::optional clusterSize) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(kernel), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(clusterSize)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaunchFuncOp LaunchFuncOp::create(::mlir::ImplicitLocOpBuilder &builder, SymbolRefAttr kernel, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Type asyncTokenType, ValueRange asyncDependencies, std::optional clusterSize) { return create(builder, builder.getLoc(), std::forward(kernel), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(clusterSize)); } LaunchFuncOp LaunchFuncOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, SymbolRefAttr kernel, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Value asyncObject, std::optional clusterSize) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(kernel), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncObject), std::forward(clusterSize)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaunchFuncOp LaunchFuncOp::create(::mlir::ImplicitLocOpBuilder &builder, SymbolRefAttr kernel, KernelDim3 gridSize, KernelDim3 blockSize, Value dynamicSharedMemorySize, ValueRange kernelOperands, Value asyncObject, std::optional clusterSize) { return create(builder, builder.getLoc(), std::forward(kernel), std::forward(gridSize), std::forward(blockSize), std::forward(dynamicSharedMemorySize), std::forward(kernelOperands), std::forward(asyncObject), std::forward(clusterSize)); } ::llvm::LogicalResult LaunchFuncOp::verifyInvariantsImpl() { auto tblgen_cooperative = getProperties().cooperative; (void)tblgen_cooperative; auto tblgen_kernel = getProperties().kernel; (void)tblgen_kernel; if (!tblgen_kernel) return emitOpError("requires attribute 'kernel'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps16(*this, tblgen_kernel, "kernel"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_cooperative, "cooperative"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup7 = getODSOperands(7); if (valueGroup7.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup7.size(); } for (auto v : valueGroup7) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup8 = getODSOperands(8); if (valueGroup8.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup8.size(); } for (auto v : valueGroup8) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup9 = getODSOperands(9); if (valueGroup9.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup9.size(); } for (auto v : valueGroup9) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup10 = getODSOperands(10); if (valueGroup10.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup10.size(); } for (auto v : valueGroup10) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup11 = getODSOperands(11); for (auto v : valueGroup11) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup12 = getODSOperands(12); if (valueGroup12.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup12.size(); } for (auto v : valueGroup12) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(2).begin()).getType()) && ((*this->getODSOperands(2).begin()).getType()) == ((*this->getODSOperands(3).begin()).getType()) && ((*this->getODSOperands(3).begin()).getType()) == ((*this->getODSOperands(4).begin()).getType()) && ((*this->getODSOperands(4).begin()).getType()) == ((*this->getODSOperands(5).begin()).getType()) && ((*this->getODSOperands(5).begin()).getType()) == ((*this->getODSOperands(6).begin()).getType()) && ((*this->getODSOperands(6).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType())))) return emitOpError("failed to verify that all of {gridSizeX, gridSizeY, gridSizeZ, blockSizeX, blockSizeY, blockSizeZ} have same type"); return ::mlir::success(); } ::llvm::LogicalResult LaunchFuncOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult LaunchFuncOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncObjectOperands; ::llvm::SMLoc asyncObjectOperandsLoc; (void)asyncObjectOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> asyncObjectTypes; ::mlir::SymbolRefAttr kernelAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> clusterSizeXOperands; ::llvm::SMLoc clusterSizeXOperandsLoc; (void)clusterSizeXOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> clusterSizeYOperands; ::llvm::SMLoc clusterSizeYOperandsLoc; (void)clusterSizeYOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> clusterSizeZOperands; ::llvm::SMLoc clusterSizeZOperandsLoc; (void)clusterSizeZOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand gridSizeXRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> gridSizeXOperands(&gridSizeXRawOperand, 1); ::llvm::SMLoc gridSizeXOperandsLoc; (void)gridSizeXOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand gridSizeYRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> gridSizeYOperands(&gridSizeYRawOperand, 1); ::llvm::SMLoc gridSizeYOperandsLoc; (void)gridSizeYOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand gridSizeZRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> gridSizeZOperands(&gridSizeZRawOperand, 1); ::llvm::SMLoc gridSizeZOperandsLoc; (void)gridSizeZOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand blockSizeXRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> blockSizeXOperands(&blockSizeXRawOperand, 1); ::llvm::SMLoc blockSizeXOperandsLoc; (void)blockSizeXOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand blockSizeYRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> blockSizeYOperands(&blockSizeYRawOperand, 1); ::llvm::SMLoc blockSizeYOperandsLoc; (void)blockSizeYOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand blockSizeZRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> blockSizeZOperands(&blockSizeZRawOperand, 1); ::llvm::SMLoc blockSizeZOperandsLoc; (void)blockSizeZOperandsLoc; ::mlir::Type gridSizeXRawType{}; ::llvm::ArrayRef<::mlir::Type> gridSizeXTypes(&gridSizeXRawType, 1); ::llvm::SmallVector<::mlir::Type, 1> clusterSizeXTypes; ::llvm::SmallVector<::mlir::Type, 1> clusterSizeYTypes; ::llvm::SmallVector<::mlir::Type, 1> clusterSizeZTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> dynamicSharedMemorySizeOperands; ::llvm::SMLoc dynamicSharedMemorySizeOperandsLoc; (void)dynamicSharedMemorySizeOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> kernelOperandsOperands; ::llvm::SMLoc kernelOperandsOperandsLoc; (void)kernelOperandsOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> kernelOperandsTypes; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } if (::mlir::succeeded(parser.parseOptionalLess())) { { asyncObjectOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); asyncObjectOperands.push_back(operand); } } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type optionalType; ::mlir::OptionalParseResult parseResult = parser.parseOptionalType(optionalType); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); asyncObjectTypes.push_back(optionalType); } } if (parser.parseGreater()) return ::mlir::failure(); } if (parser.parseCustomAttributeWithFallback(kernelAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (kernelAttr) result.getOrAddProperties().kernel = kernelAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("clusters"))) { if (parser.parseKeyword("in")) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); { clusterSizeXOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); clusterSizeXOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { clusterSizeYOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); clusterSizeYOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { clusterSizeZOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); clusterSizeZOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } if (parser.parseKeyword("blocks")) return ::mlir::failure(); if (parser.parseKeyword("in")) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); gridSizeXOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(gridSizeXRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); gridSizeYOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(gridSizeYRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); gridSizeZOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(gridSizeZRawOperand)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); if (parser.parseKeyword("threads")) return ::mlir::failure(); if (parser.parseKeyword("in")) return ::mlir::failure(); if (parser.parseLParen()) return ::mlir::failure(); blockSizeXOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(blockSizeXRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); blockSizeYOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(blockSizeYRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); blockSizeZOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(blockSizeZRawOperand)) return ::mlir::failure(); if (parser.parseRParen()) return ::mlir::failure(); { ::std::optional<::mlir::OpAsmParser::UnresolvedOperand> clusterSizeXOperand = clusterSizeXOperands.empty() ? ::std::optional<::mlir::OpAsmParser::UnresolvedOperand>() : clusterSizeXOperands[0]; ::mlir::Type clusterSizeXType; ::mlir::Type clusterSizeYType; ::mlir::Type clusterSizeZType; auto odsResult = parseLaunchDimType(parser, gridSizeXRawType, clusterSizeXOperand, clusterSizeXType, clusterSizeYType, clusterSizeZType); if (odsResult) return ::mlir::failure(); if (clusterSizeXType) clusterSizeXTypes.push_back(clusterSizeXType); if (clusterSizeYType) clusterSizeYTypes.push_back(clusterSizeYType); if (clusterSizeZType) clusterSizeZTypes.push_back(clusterSizeZType); } if (::mlir::succeeded(parser.parseOptionalKeyword("dynamic_shared_memory_size"))) { { dynamicSharedMemorySizeOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); dynamicSharedMemorySizeOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalKeyword("cooperative"))) { result.getOrAddProperties().cooperative = parser.getBuilder().getUnitAttr(); } { kernelOperandsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseLaunchFuncOperands(parser, kernelOperandsOperands, kernelOperandsTypes); if (odsResult) return ::mlir::failure(); } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependenciesOperands.size()), 1, 1, 1, 1, 1, 1, static_cast(clusterSizeXOperands.size()), static_cast(clusterSizeYOperands.size()), static_cast(clusterSizeZOperands.size()), static_cast(dynamicSharedMemorySizeOperands.size()), static_cast(kernelOperandsOperands.size()), static_cast(asyncObjectOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(32); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(gridSizeXOperands, gridSizeXTypes, gridSizeXOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(gridSizeYOperands, gridSizeXTypes[0], gridSizeYOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(gridSizeZOperands, gridSizeXTypes[0], gridSizeZOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(blockSizeXOperands, gridSizeXTypes[0], blockSizeXOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(blockSizeYOperands, gridSizeXTypes[0], blockSizeYOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(blockSizeZOperands, gridSizeXTypes[0], blockSizeZOperandsLoc, result.operands)) return ::mlir::failure(); if (!clusterSizeXOperands.empty()) { if (parser.resolveOperands(clusterSizeXOperands, clusterSizeXTypes, clusterSizeXOperandsLoc, result.operands)) return ::mlir::failure(); } if (!clusterSizeYOperands.empty()) { if (parser.resolveOperands(clusterSizeYOperands, clusterSizeYTypes, clusterSizeYOperandsLoc, result.operands)) return ::mlir::failure(); } if (!clusterSizeZOperands.empty()) { if (parser.resolveOperands(clusterSizeZOperands, clusterSizeZTypes, clusterSizeZOperandsLoc, result.operands)) return ::mlir::failure(); } if (!dynamicSharedMemorySizeOperands.empty()) { if (parser.resolveOperands(dynamicSharedMemorySizeOperands, odsBuildableType1, dynamicSharedMemorySizeOperandsLoc, result.operands)) return ::mlir::failure(); } if (parser.resolveOperands(kernelOperandsOperands, kernelOperandsTypes, kernelOperandsOperandsLoc, result.operands)) return ::mlir::failure(); if (!asyncObjectOperands.empty()) { if (parser.resolveOperands(asyncObjectOperands, asyncObjectTypes, asyncObjectOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void LaunchFuncOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); if (getAsyncObject()) { _odsPrinter << "<"; if (::mlir::Value value = getAsyncObject()) _odsPrinter << value; _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << (getAsyncObject() ? ::llvm::ArrayRef<::mlir::Type>(getAsyncObject().getType()) : ::llvm::ArrayRef<::mlir::Type>()); _odsPrinter << ">"; } _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getKernelAttr()); if (getClusterSizeX()) { _odsPrinter << ' ' << "clusters"; _odsPrinter << ' ' << "in"; _odsPrinter << ' '; _odsPrinter << "("; if (::mlir::Value value = getClusterSizeX()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getClusterSizeY()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getClusterSizeZ()) _odsPrinter << value; _odsPrinter << ")"; } _odsPrinter << ' ' << "blocks"; _odsPrinter << ' ' << "in"; _odsPrinter << ' '; _odsPrinter << "("; _odsPrinter << getGridSizeX(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getGridSizeY(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getGridSizeZ(); _odsPrinter << ")"; _odsPrinter << ' ' << "threads"; _odsPrinter << ' ' << "in"; _odsPrinter << ' '; _odsPrinter << "("; _odsPrinter << getBlockSizeX(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBlockSizeY(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBlockSizeZ(); _odsPrinter << ")"; _odsPrinter << ' '; printLaunchDimType(_odsPrinter, *this, getGridSizeX().getType(), getClusterSizeX(), (getClusterSizeX() ? getClusterSizeX().getType() : ::mlir::Type()), (getClusterSizeY() ? getClusterSizeY().getType() : ::mlir::Type()), (getClusterSizeZ() ? getClusterSizeZ().getType() : ::mlir::Type())); if (getDynamicSharedMemorySize()) { _odsPrinter << ' ' << "dynamic_shared_memory_size"; _odsPrinter << ' '; if (::mlir::Value value = getDynamicSharedMemorySize()) _odsPrinter << value; } if ((getCooperativeAttr() && getCooperativeAttr() != ((false) ? ::mlir::OpBuilder((*this)->getContext()).getUnitAttr() : nullptr))) { _odsPrinter << ' ' << "cooperative"; } _odsPrinter << ' '; printLaunchFuncOperands(_odsPrinter, *this, getKernelOperands(), getKernelOperands().getTypes()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("kernel"); elidedAttrs.push_back("cooperative"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getCooperativeAttr(); if(attr && (attr == ((false) ? odsBuilder.getUnitAttr() : nullptr))) elidedAttrs.push_back("cooperative"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::LaunchFuncOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::LaunchFuncOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::LaunchOp definitions //===----------------------------------------------------------------------===// namespace detail { LaunchOpGenericAdaptorBase::LaunchOpGenericAdaptorBase(LaunchOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LaunchOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::UnitAttr LaunchOpGenericAdaptorBase::getCooperativeAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().cooperative); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool LaunchOpGenericAdaptorBase::getCooperative() { auto attr = getCooperativeAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } ::std::optional< ::llvm::StringRef > LaunchOpGenericAdaptorBase::getModule() { auto attr = getModuleAttr(); return attr ? ::std::optional< ::llvm::StringRef >(attr.getValue()) : (::std::nullopt); } ::std::optional< ::llvm::StringRef > LaunchOpGenericAdaptorBase::getFunction() { auto attr = getFunctionAttr(); return attr ? ::std::optional< ::llvm::StringRef >(attr.getValue()) : (::std::nullopt); } ::std::optional LaunchOpGenericAdaptorBase::getWorkgroupAttributions() { auto attr = getWorkgroupAttributionsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } } // namespace detail LaunchOpAdaptor::LaunchOpAdaptor(LaunchOp op) : LaunchOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LaunchOpAdaptor::verify(::mlir::Location loc) { auto tblgen_cooperative = getProperties().cooperative; (void)tblgen_cooperative; auto tblgen_function = getProperties().function; (void)tblgen_function; auto tblgen_module = getProperties().module; (void)tblgen_module; auto tblgen_workgroup_attributions = getProperties().workgroup_attributions; (void)tblgen_workgroup_attributions; if (tblgen_cooperative && !((::llvm::isa<::mlir::UnitAttr>(tblgen_cooperative)))) return emitError(loc, "'gpu.launch' op attribute 'cooperative' failed to satisfy constraint: unit attribute"); if (tblgen_module && !((::llvm::isa<::mlir::FlatSymbolRefAttr>(tblgen_module)))) return emitError(loc, "'gpu.launch' op attribute 'module' failed to satisfy constraint: flat symbol reference attribute"); if (tblgen_function && !((::llvm::isa<::mlir::FlatSymbolRefAttr>(tblgen_function)))) return emitError(loc, "'gpu.launch' op attribute 'function' failed to satisfy constraint: flat symbol reference attribute"); if (tblgen_workgroup_attributions && !((((::llvm::isa<::mlir::IntegerAttr>(tblgen_workgroup_attributions))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_workgroup_attributions).getType().isSignlessInteger(64)))) && ((!::llvm::cast<::mlir::IntegerAttr>(tblgen_workgroup_attributions).getValue().isNegative())))) return emitError(loc, "'gpu.launch' op attribute 'workgroup_attributions' failed to satisfy constraint: 64-bit signless integer attribute whose value is non-negative"); return ::mlir::success(); } std::pair LaunchOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange LaunchOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(0u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchOp::getClusterSizeXMutable() { auto range = getODSOperandIndexAndLength(7); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(7u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchOp::getClusterSizeYMutable() { auto range = getODSOperandIndexAndLength(8); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(8u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchOp::getClusterSizeZMutable() { auto range = getODSOperandIndexAndLength(9); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(9u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange LaunchOp::getDynamicSharedMemorySizeMutable() { auto range = getODSOperandIndexAndLength(10); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(10u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } std::pair LaunchOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult LaunchOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.cooperative; auto attr = dict.get("cooperative"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `cooperative` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.function; auto attr = dict.get("function"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `function` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.module; auto attr = dict.get("module"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `module` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attributions; auto attr = dict.get("workgroup_attributions"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attributions` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute LaunchOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.cooperative; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("cooperative", propStorage)); } { const auto &propStorage = prop.function; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("function", propStorage)); } { const auto &propStorage = prop.module; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("module", propStorage)); } { const auto &propStorage = prop.workgroup_attributions; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("workgroup_attributions", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LaunchOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.cooperative.getAsOpaquePointer()), llvm::hash_value(prop.function.getAsOpaquePointer()), llvm::hash_value(prop.module.getAsOpaquePointer()), llvm::hash_value(prop.workgroup_attributions.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional LaunchOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "cooperative") return prop.cooperative; if (name == "function") return prop.function; if (name == "module") return prop.module; if (name == "workgroup_attributions") return prop.workgroup_attributions; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void LaunchOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "cooperative") { prop.cooperative = ::llvm::dyn_cast_or_null>(value); return; } if (name == "function") { prop.function = ::llvm::dyn_cast_or_null>(value); return; } if (name == "module") { prop.module = ::llvm::dyn_cast_or_null>(value); return; } if (name == "workgroup_attributions") { prop.workgroup_attributions = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void LaunchOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.cooperative) attrs.append("cooperative", prop.cooperative); if (prop.function) attrs.append("function", prop.function); if (prop.module) attrs.append("module", prop.module); if (prop.workgroup_attributions) attrs.append("workgroup_attributions", prop.workgroup_attributions); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult LaunchOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getCooperativeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "cooperative", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getFunctionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps17(attr, "function", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModuleAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps17(attr, "module", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getWorkgroupAttributionsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps14(attr, "workgroup_attributions", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LaunchOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.cooperative))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.function))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.module))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } if (::mlir::failed(reader.readOptionalAttribute(prop.workgroup_attributions))) return ::mlir::failure(); { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void LaunchOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.cooperative); writer.writeOptionalAttribute(prop.function); writer.writeOptionalAttribute(prop.module); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } writer.writeOptionalAttribute(prop.workgroup_attributions); { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } bool LaunchOp::getCooperative() { auto attr = getCooperativeAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } ::std::optional< ::llvm::StringRef > LaunchOp::getModule() { auto attr = getModuleAttr(); return attr ? ::std::optional< ::llvm::StringRef >(attr.getValue()) : (::std::nullopt); } ::std::optional< ::llvm::StringRef > LaunchOp::getFunction() { auto attr = getFunctionAttr(); return attr ? ::std::optional< ::llvm::StringRef >(attr.getValue()) : (::std::nullopt); } ::std::optional LaunchOp::getWorkgroupAttributions() { auto attr = getWorkgroupAttributionsAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } void LaunchOp::setCooperative(bool attrValue) { auto &odsProp = getProperties().cooperative; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void LaunchOp::setModule(::std::optional<::llvm::StringRef> attrValue) { auto &odsProp = getProperties().module; if (attrValue) odsProp = ::mlir::SymbolRefAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LaunchOp::setFunction(::std::optional<::llvm::StringRef> attrValue) { auto &odsProp = getProperties().function; if (attrValue) odsProp = ::mlir::SymbolRefAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LaunchOp::setWorkgroupAttributions(::std::optional attrValue) { auto &odsProp = getProperties().workgroup_attributions; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } LaunchOp LaunchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value gridSizeX, Value gridSizeY, Value gridSizeZ, Value blockSizeX, Value blockSizeY, Value blockSizeZ, Value dynamicSharedMemorySize, Type asyncTokenType, ValueRange asyncDependencies, TypeRange workgroupAttributions, TypeRange privateAttributions, Value clusterSizeX, Value clusterSizeY, Value clusterSizeZ, FlatSymbolRefAttr module, FlatSymbolRefAttr function) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(gridSizeX), std::forward(gridSizeY), std::forward(gridSizeZ), std::forward(blockSizeX), std::forward(blockSizeY), std::forward(blockSizeZ), std::forward(dynamicSharedMemorySize), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(workgroupAttributions), std::forward(privateAttributions), std::forward(clusterSizeX), std::forward(clusterSizeY), std::forward(clusterSizeZ), std::forward(module), std::forward(function)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LaunchOp LaunchOp::create(::mlir::ImplicitLocOpBuilder &builder, Value gridSizeX, Value gridSizeY, Value gridSizeZ, Value blockSizeX, Value blockSizeY, Value blockSizeZ, Value dynamicSharedMemorySize, Type asyncTokenType, ValueRange asyncDependencies, TypeRange workgroupAttributions, TypeRange privateAttributions, Value clusterSizeX, Value clusterSizeY, Value clusterSizeZ, FlatSymbolRefAttr module, FlatSymbolRefAttr function) { return create(builder, builder.getLoc(), std::forward(gridSizeX), std::forward(gridSizeY), std::forward(gridSizeZ), std::forward(blockSizeX), std::forward(blockSizeY), std::forward(blockSizeZ), std::forward(dynamicSharedMemorySize), std::forward(asyncTokenType), std::forward(asyncDependencies), std::forward(workgroupAttributions), std::forward(privateAttributions), std::forward(clusterSizeX), std::forward(clusterSizeY), std::forward(clusterSizeZ), std::forward(module), std::forward(function)); } ::llvm::LogicalResult LaunchOp::verifyInvariantsImpl() { auto tblgen_cooperative = getProperties().cooperative; (void)tblgen_cooperative; auto tblgen_function = getProperties().function; (void)tblgen_function; auto tblgen_module = getProperties().module; (void)tblgen_module; auto tblgen_workgroup_attributions = getProperties().workgroup_attributions; (void)tblgen_workgroup_attributions; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_cooperative, "cooperative"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps17(*this, tblgen_module, "module"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps17(*this, tblgen_function, "function"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps14(*this, tblgen_workgroup_attributions, "workgroup_attributions"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup7 = getODSOperands(7); if (valueGroup7.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup7.size(); } for (auto v : valueGroup7) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup8 = getODSOperands(8); if (valueGroup8.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup8.size(); } for (auto v : valueGroup8) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup9 = getODSOperands(9); if (valueGroup9.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup9.size(); } for (auto v : valueGroup9) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup10 = getODSOperands(10); if (valueGroup10.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup10.size(); } for (auto v : valueGroup10) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; for (auto ®ion : ::llvm::MutableArrayRef((*this)->getRegion(0))) if (::mlir::failed(__mlir_ods_local_region_constraint_GPUOps1(*this, region, "body", index++))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LaunchOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult LaunchOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.cooperative; auto cooperativeAttrName = ::mlir::StringAttr::get(ctx, "cooperative"); auto attr = dict.get(cooperativeAttrName); usedKeys.insert(cooperativeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for cooperative in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `cooperative` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.module; auto moduleAttrName = ::mlir::StringAttr::get(ctx, "module"); auto attr = dict.get(moduleAttrName); usedKeys.insert(moduleAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for module in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `module` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.function; auto functionAttrName = ::mlir::StringAttr::get(ctx, "function"); auto attr = dict.get(functionAttrName); usedKeys.insert(functionAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for function in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `function` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.workgroup_attributions; auto workgroup_attributionsAttrName = ::mlir::StringAttr::get(ctx, "workgroup_attributions"); auto attr = dict.get(workgroup_attributionsAttrName); usedKeys.insert(workgroup_attributionsAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for workgroup_attributions in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `workgroup_attributions` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::LaunchOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::LaunchOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::MemcpyOp definitions //===----------------------------------------------------------------------===// namespace detail { MemcpyOpGenericAdaptorBase::MemcpyOpGenericAdaptorBase(MemcpyOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair MemcpyOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail MemcpyOpAdaptor::MemcpyOpAdaptor(MemcpyOp op) : MemcpyOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MemcpyOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair MemcpyOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange MemcpyOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair MemcpyOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult MemcpyOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MemcpyOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MemcpyOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MemcpyOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MemcpyOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MemcpyOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MemcpyOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MemcpyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { odsState.addOperands(asyncDependencies); odsState.addOperands(dst); odsState.addOperands(src); if (asyncToken) odsState.addTypes(asyncToken); } MemcpyOp MemcpyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dst), std::forward(src)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemcpyOp MemcpyOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dst), std::forward(src)); } void MemcpyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { odsState.addOperands(asyncDependencies); odsState.addOperands(dst); odsState.addOperands(src); odsState.addTypes(resultTypes); } MemcpyOp MemcpyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dst), std::forward(src)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemcpyOp MemcpyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value src) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dst), std::forward(src)); } void MemcpyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } MemcpyOp MemcpyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemcpyOp MemcpyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MemcpyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } MemcpyOp MemcpyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemcpyOp MemcpyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MemcpyOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult MemcpyOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult MemcpyOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc; (void)dstOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc; (void)srcOperandsLoc; ::mlir::Type dstRawType{}; ::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1); ::mlir::Type srcRawType{}; ::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } dstOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); srcOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcRawOperand)) return ::mlir::failure(); if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcRawType = type; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MemcpyOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDst(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSrc(); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getDst().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getSrc().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void MemcpyOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } { auto valueRange = getODSOperandIndexAndLength(2); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::MemcpyOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::MemsetOp definitions //===----------------------------------------------------------------------===// namespace detail { MemsetOpGenericAdaptorBase::MemsetOpGenericAdaptorBase(MemsetOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair MemsetOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail MemsetOpAdaptor::MemsetOpAdaptor(MemsetOp op) : MemsetOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult MemsetOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair MemsetOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange MemsetOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair MemsetOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult MemsetOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute MemsetOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code MemsetOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional MemsetOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void MemsetOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void MemsetOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult MemsetOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void MemsetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { odsState.addOperands(asyncDependencies); odsState.addOperands(dst); odsState.addOperands(value); if (asyncToken) odsState.addTypes(asyncToken); } MemsetOp MemsetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dst), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemsetOp MemsetOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dst), std::forward(value)); } void MemsetOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { odsState.addOperands(asyncDependencies); odsState.addOperands(dst); odsState.addOperands(value); odsState.addTypes(resultTypes); } MemsetOp MemsetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dst), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemsetOp MemsetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value dst, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(dst), std::forward(value)); } void MemsetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } MemsetOp MemsetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemsetOp MemsetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void MemsetOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } MemsetOp MemsetOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } MemsetOp MemsetOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult MemsetOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin())))))) return emitOpError("failed to verify that all of {dst, value} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult MemsetOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult MemsetOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstOperands(&dstRawOperand, 1); ::llvm::SMLoc dstOperandsLoc; (void)dstOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::Type dstRawType{}; ::llvm::ArrayRef<::mlir::Type> dstTypes(&dstRawType, 1); ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } dstOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dstOperands, dstTypes, dstOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void MemsetOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDst(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getDst().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void MemsetOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::MemsetOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::NumSubgroupsOp definitions //===----------------------------------------------------------------------===// namespace detail { NumSubgroupsOpGenericAdaptorBase::NumSubgroupsOpGenericAdaptorBase(NumSubgroupsOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional< ::llvm::APInt > NumSubgroupsOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail NumSubgroupsOpAdaptor::NumSubgroupsOpAdaptor(NumSubgroupsOp op) : NumSubgroupsOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult NumSubgroupsOpAdaptor::verify(::mlir::Location loc) { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.num_subgroups' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult NumSubgroupsOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute NumSubgroupsOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code NumSubgroupsOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional NumSubgroupsOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void NumSubgroupsOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void NumSubgroupsOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult NumSubgroupsOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult NumSubgroupsOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void NumSubgroupsOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.upper_bound); } ::std::optional< ::llvm::APInt > NumSubgroupsOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void NumSubgroupsOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void NumSubgroupsOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(result); } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(result), std::forward(upper_bound)); } void NumSubgroupsOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(NumSubgroupsOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(upper_bound)); } void NumSubgroupsOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(upper_bound)); } void NumSubgroupsOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void NumSubgroupsOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(NumSubgroupsOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void NumSubgroupsOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void NumSubgroupsOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(NumSubgroupsOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } NumSubgroupsOp NumSubgroupsOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NumSubgroupsOp NumSubgroupsOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult NumSubgroupsOp::verifyInvariantsImpl() { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult NumSubgroupsOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult NumSubgroupsOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult NumSubgroupsOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::IntegerAttr upper_boundAttr; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IndexType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); return ::mlir::success(); } void NumSubgroupsOp::print(::mlir::OpAsmPrinter &_odsPrinter) { if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IndexType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void NumSubgroupsOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::NumSubgroupsOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::NumSubgroupsOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::PrintfOp definitions //===----------------------------------------------------------------------===// namespace detail { PrintfOpGenericAdaptorBase::PrintfOpGenericAdaptorBase(PrintfOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair PrintfOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::StringRef PrintfOpGenericAdaptorBase::getFormat() { auto attr = getFormatAttr(); return attr.getValue(); } } // namespace detail PrintfOpAdaptor::PrintfOpAdaptor(PrintfOp op) : PrintfOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult PrintfOpAdaptor::verify(::mlir::Location loc) { auto tblgen_format = getProperties().format; (void)tblgen_format; if (!tblgen_format) return emitError(loc, "'gpu.printf' op requires attribute 'format'"); if (tblgen_format && !((::llvm::isa<::mlir::StringAttr>(tblgen_format)))) return emitError(loc, "'gpu.printf' op attribute 'format' failed to satisfy constraint: string attribute"); return ::mlir::success(); } std::pair PrintfOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange PrintfOp::getArgsMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult PrintfOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.format; auto attr = dict.get("format"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `format` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute PrintfOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.format; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("format", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code PrintfOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.format.getAsOpaquePointer())); } std::optional PrintfOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "format") return prop.format; return std::nullopt; } void PrintfOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "format") { prop.format = ::llvm::dyn_cast_or_null>(value); return; } } void PrintfOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.format) attrs.append("format", prop.format); } ::llvm::LogicalResult PrintfOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getFormatAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(attr, "format", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult PrintfOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.format))) return ::mlir::failure(); return ::mlir::success(); } void PrintfOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.format); } ::llvm::StringRef PrintfOp::getFormat() { auto attr = getFormatAttr(); return attr.getValue(); } void PrintfOp::setFormat(::llvm::StringRef attrValue) { getProperties().format = ::mlir::Builder((*this)->getContext()).getStringAttr(attrValue); } void PrintfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::StringAttr format, ::mlir::ValueRange args) { odsState.addOperands(args); odsState.getOrAddProperties().format = format; } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::StringAttr format, ::mlir::ValueRange args) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(format), std::forward(args)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::StringAttr format, ::mlir::ValueRange args) { return create(builder, builder.getLoc(), std::forward(format), std::forward(args)); } void PrintfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::StringAttr format, ::mlir::ValueRange args) { odsState.addOperands(args); odsState.getOrAddProperties().format = format; assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::StringAttr format, ::mlir::ValueRange args) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(format), std::forward(args)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::StringAttr format, ::mlir::ValueRange args) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(format), std::forward(args)); } void PrintfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::llvm::StringRef format, ::mlir::ValueRange args) { odsState.addOperands(args); odsState.getOrAddProperties().format = odsBuilder.getStringAttr(format); } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::llvm::StringRef format, ::mlir::ValueRange args) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(format), std::forward(args)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::llvm::StringRef format, ::mlir::ValueRange args) { return create(builder, builder.getLoc(), std::forward(format), std::forward(args)); } void PrintfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::llvm::StringRef format, ::mlir::ValueRange args) { odsState.addOperands(args); odsState.getOrAddProperties().format = odsBuilder.getStringAttr(format); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::llvm::StringRef format, ::mlir::ValueRange args) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(format), std::forward(args)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::llvm::StringRef format, ::mlir::ValueRange args) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(format), std::forward(args)); } void PrintfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void PrintfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } PrintfOp PrintfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrintfOp PrintfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult PrintfOp::verifyInvariantsImpl() { auto tblgen_format = getProperties().format; (void)tblgen_format; if (!tblgen_format) return emitOpError("requires attribute 'format'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps5(*this, tblgen_format, "format"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps18(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult PrintfOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult PrintfOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::StringAttr formatAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> argsOperands; ::llvm::SMLoc argsOperandsLoc; (void)argsOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> argsTypes; if (parser.parseCustomAttributeWithFallback(formatAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (formatAttr) result.getOrAddProperties().format = formatAttr; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (::mlir::succeeded(parser.parseOptionalComma())) { argsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(argsOperands)) return ::mlir::failure(); if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(argsTypes)) return ::mlir::failure(); } if (parser.resolveOperands(argsOperands, argsTypes, argsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void PrintfOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getFormatAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("format"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); if (!getArgs().empty()) { _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getArgs(); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getArgs().getTypes(); } } void PrintfOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::PrintfOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::PrintfOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ReturnOp definitions //===----------------------------------------------------------------------===// namespace detail { ReturnOpGenericAdaptorBase::ReturnOpGenericAdaptorBase(ReturnOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair ReturnOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail ReturnOpAdaptor::ReturnOpAdaptor(ReturnOp op) : ReturnOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ReturnOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair ReturnOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange ReturnOp::getOperandsMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult ReturnOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute ReturnOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ReturnOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional ReturnOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void ReturnOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void ReturnOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult ReturnOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void ReturnOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState) { // empty } ReturnOp ReturnOp::create(::mlir::OpBuilder &builder, ::mlir::Location location) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ReturnOp ReturnOp::create(::mlir::ImplicitLocOpBuilder &builder) { return create(builder, builder.getLoc()); } void ReturnOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands) { odsState.addOperands(operands); } ReturnOp ReturnOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ReturnOp ReturnOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands) { return create(builder, builder.getLoc(), std::forward(operands)); } void ReturnOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ReturnOp ReturnOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ReturnOp ReturnOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ReturnOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ReturnOp ReturnOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ReturnOp ReturnOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ReturnOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ReturnOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult ReturnOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> operandsOperands; ::llvm::SMLoc operandsOperandsLoc; (void)operandsOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> operandsTypes; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } operandsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(operandsOperands)) return ::mlir::failure(); if (!operandsOperands.empty()) { if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(operandsTypes)) return ::mlir::failure(); } if (parser.resolveOperands(operandsOperands, operandsTypes, operandsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void ReturnOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); if (!getOperands().empty()) { _odsPrinter << ' '; _odsPrinter << getOperands(); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getOperands().getTypes(); } } void ReturnOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ReturnOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::RotateOp definitions //===----------------------------------------------------------------------===// namespace detail { RotateOpGenericAdaptorBase::RotateOpGenericAdaptorBase(RotateOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} uint32_t RotateOpGenericAdaptorBase::getOffset() { auto attr = getOffsetAttr(); return attr.getValue().getZExtValue(); } uint32_t RotateOpGenericAdaptorBase::getWidth() { auto attr = getWidthAttr(); return attr.getValue().getZExtValue(); } } // namespace detail RotateOpAdaptor::RotateOpAdaptor(RotateOp op) : RotateOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult RotateOpAdaptor::verify(::mlir::Location loc) { auto tblgen_offset = getProperties().offset; (void)tblgen_offset; if (!tblgen_offset) return emitError(loc, "'gpu.rotate' op requires attribute 'offset'"); auto tblgen_width = getProperties().width; (void)tblgen_width; if (!tblgen_width) return emitError(loc, "'gpu.rotate' op requires attribute 'width'"); if (tblgen_offset && !((((::llvm::isa<::mlir::IntegerAttr>(tblgen_offset))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_offset).getType().isSignlessInteger(32)))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_offset).getInt() >= 0)))) return emitError(loc, "'gpu.rotate' op attribute 'offset' failed to satisfy constraint: 32-bit signless integer attribute whose minimum value is 0"); if (tblgen_width && !((((::llvm::isa<::mlir::IntegerAttr>(tblgen_width))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_width).getType().isSignlessInteger(32)))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_width).getValue().isPowerOf2())))) return emitError(loc, "'gpu.rotate' op attribute 'width' failed to satisfy constraint: 32-bit signless integer attribute whose value is a power of two > 0"); return ::mlir::success(); } void RotateOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "rotateResult"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "valid"); } ::llvm::LogicalResult RotateOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.offset; auto attr = dict.get("offset"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `offset` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.width; auto attr = dict.get("width"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `width` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute RotateOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.offset; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("offset", propStorage)); } { const auto &propStorage = prop.width; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("width", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code RotateOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.offset.getAsOpaquePointer()), llvm::hash_value(prop.width.getAsOpaquePointer())); } std::optional RotateOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "offset") return prop.offset; if (name == "width") return prop.width; return std::nullopt; } void RotateOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "offset") { prop.offset = ::llvm::dyn_cast_or_null>(value); return; } if (name == "width") { prop.width = ::llvm::dyn_cast_or_null>(value); return; } } void RotateOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.offset) attrs.append("offset", prop.offset); if (prop.width) attrs.append("width", prop.width); } ::llvm::LogicalResult RotateOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getOffsetAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps18(attr, "offset", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getWidthAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps19(attr, "width", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult RotateOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.offset))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.width))) return ::mlir::failure(); return ::mlir::success(); } void RotateOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.offset); writer.writeAttribute(prop.width); } uint32_t RotateOp::getOffset() { auto attr = getOffsetAttr(); return attr.getValue().getZExtValue(); } uint32_t RotateOp::getWidth() { auto attr = getWidthAttr(); return attr.getValue().getZExtValue(); } void RotateOp::setOffset(uint32_t attrValue) { getProperties().offset = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue); } void RotateOp::setWidth(uint32_t attrValue) { getProperties().width = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = offset; odsState.getOrAddProperties().width = width; odsState.addTypes(rotateResult); odsState.addTypes(valid); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(rotateResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { return create(builder, builder.getLoc(), std::forward(rotateResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = offset; odsState.getOrAddProperties().width = width; ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RotateOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { return create(builder, builder.getLoc(), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = offset; odsState.getOrAddProperties().width = width; assert(resultTypes.size() == 2u && "mismatched number of results"); odsState.addTypes(resultTypes); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::IntegerAttr offset, ::mlir::IntegerAttr width) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, uint32_t offset, uint32_t width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), offset); odsState.getOrAddProperties().width = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), width); odsState.addTypes(rotateResult); odsState.addTypes(valid); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, uint32_t offset, uint32_t width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(rotateResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type rotateResult, ::mlir::Type valid, ::mlir::Value value, uint32_t offset, uint32_t width) { return create(builder, builder.getLoc(), std::forward(rotateResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, uint32_t offset, uint32_t width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), offset); odsState.getOrAddProperties().width = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), width); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RotateOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, uint32_t offset, uint32_t width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, uint32_t offset, uint32_t width) { return create(builder, builder.getLoc(), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, uint32_t offset, uint32_t width) { odsState.addOperands(value); odsState.getOrAddProperties().offset = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), offset); odsState.getOrAddProperties().width = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), width); assert(resultTypes.size() == 2u && "mismatched number of results"); odsState.addTypes(resultTypes); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, uint32_t offset, uint32_t width) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, uint32_t offset, uint32_t width) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width)); } void RotateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RotateOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void RotateOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(resultTypes); } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void RotateOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(RotateOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } RotateOp RotateOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } RotateOp RotateOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult RotateOp::verifyInvariantsImpl() { auto tblgen_offset = getProperties().offset; (void)tblgen_offset; if (!tblgen_offset) return emitOpError("requires attribute 'offset'"); auto tblgen_width = getProperties().width; (void)tblgen_width; if (!tblgen_width) return emitOpError("requires attribute 'width'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps18(*this, tblgen_offset, "offset"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps19(*this, tblgen_width, "width"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {value, rotateResult} have same type"); return ::mlir::success(); } ::llvm::LogicalResult RotateOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult RotateOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(2); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); ::mlir::Type odsInferredType1 = odsBuilder.getIntegerType(1); inferredReturnTypes[0] = odsInferredType0; inferredReturnTypes[1] = odsInferredType1; return ::mlir::success(); } ::mlir::ParseResult RotateOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::IntegerAttr offsetAttr; ::mlir::IntegerAttr widthAttr; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(offsetAttr, parser.getBuilder().getIntegerType(32))) { return ::mlir::failure(); } if (offsetAttr) result.getOrAddProperties().offset = offsetAttr; if (parser.parseComma()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(widthAttr, parser.getBuilder().getIntegerType(32))) { return ::mlir::failure(); } if (widthAttr) result.getOrAddProperties().width = widthAttr; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); result.addTypes(valueTypes[0]); result.addTypes(odsBuildableType0); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void RotateOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getOffsetAttr()); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getWidthAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("offset"); elidedAttrs.push_back("width"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void RotateOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::RotateOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::RotateOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SDDMMBufferSizeOp definitions //===----------------------------------------------------------------------===// namespace detail { SDDMMBufferSizeOpGenericAdaptorBase::SDDMMBufferSizeOpGenericAdaptorBase(SDDMMBufferSizeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SDDMMBufferSizeOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SDDMMBufferSizeOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SDDMMBufferSizeOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SDDMMBufferSizeOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SDDMMBufferSizeOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SDDMMBufferSizeOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SDDMMBufferSizeOpAdaptor::SDDMMBufferSizeOpAdaptor(SDDMMBufferSizeOp op) : SDDMMBufferSizeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SDDMMBufferSizeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.sddmm_buffer_size' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.sddmm_buffer_size' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.sddmm_buffer_size' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.sddmm_buffer_size' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } void SDDMMBufferSizeOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "bufferSz"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair SDDMMBufferSizeOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SDDMMBufferSizeOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SDDMMBufferSizeOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SDDMMBufferSizeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SDDMMBufferSizeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SDDMMBufferSizeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer())); } std::optional SDDMMBufferSizeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; return std::nullopt; } void SDDMMBufferSizeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } } void SDDMMBufferSizeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); } ::llvm::LogicalResult SDDMMBufferSizeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SDDMMBufferSizeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); return ::mlir::success(); } void SDDMMBufferSizeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); } ::mlir::gpu::TransposeMode SDDMMBufferSizeOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SDDMMBufferSizeOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SDDMMBufferSizeOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SDDMMBufferSizeOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SDDMMBufferSizeOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SDDMMBufferSizeOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, bufferSz, asyncToken, asyncDependencies, modeA, modeB, dnmatA, dnmatB, spmatC, computeType); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(bufferSz); if (asyncToken) odsState.addTypes(asyncToken); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(bufferSz); if (asyncToken) odsState.addTypes(asyncToken); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SDDMMBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMBufferSizeOp SDDMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SDDMMBufferSizeOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SDDMMBufferSizeOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SDDMMBufferSizeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SDDMMBufferSizeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dnmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatAOperands(&dnmatARawOperand, 1); ::llvm::SMLoc dnmatAOperandsLoc; (void)dnmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatBOperands(&dnmatBRawOperand, 1); ::llvm::SMLoc dnmatBOperandsLoc; (void)dnmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatCOperands(&spmatCRawOperand, 1); ::llvm::SMLoc spmatCOperandsLoc; (void)spmatCOperandsLoc; ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } dnmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatCRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType3 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatAOperands, odsBuildableType2, dnmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatBOperands, odsBuildableType2, dnmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatCOperands, odsBuildableType3, spmatCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SDDMMBufferSizeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDnmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatC(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SDDMMBufferSizeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SDDMMBufferSizeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SDDMMOp definitions //===----------------------------------------------------------------------===// namespace detail { SDDMMOpGenericAdaptorBase::SDDMMOpGenericAdaptorBase(SDDMMOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SDDMMOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SDDMMOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SDDMMOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SDDMMOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SDDMMOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SDDMMOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SDDMMOpAdaptor::SDDMMOpAdaptor(SDDMMOp op) : SDDMMOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SDDMMOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.sddmm' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.sddmm' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.sddmm' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.sddmm' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } std::pair SDDMMOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SDDMMOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SDDMMOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SDDMMOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SDDMMOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SDDMMOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer())); } std::optional SDDMMOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; return std::nullopt; } void SDDMMOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } } void SDDMMOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); } ::llvm::LogicalResult SDDMMOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SDDMMOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); return ::mlir::success(); } void SDDMMOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); } ::mlir::gpu::TransposeMode SDDMMOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SDDMMOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SDDMMOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SDDMMOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SDDMMOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SDDMMOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SDDMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType, Value buffer) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, asyncToken, asyncDependencies, modeA, modeB, dnmatA, dnmatB, spmatC, computeType, buffer); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType, Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, Type asyncToken, ValueRange asyncDependencies, Value dnmatA, Value dnmatB, Value spmatC, Type computeType, Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); } void SDDMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; if (asyncToken) odsState.addTypes(asyncToken); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); } void SDDMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(resultTypes); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); } void SDDMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); if (asyncToken) odsState.addTypes(asyncToken); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); } void SDDMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(dnmatA); odsState.addOperands(dnmatB); odsState.addOperands(spmatC); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(resultTypes); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value dnmatA, ::mlir::Value dnmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(dnmatA), std::forward(dnmatB), std::forward(spmatC), std::forward(computeType), std::forward(buffer)); } void SDDMMOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SDDMMOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SDDMMOp SDDMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SDDMMOp SDDMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SDDMMOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SDDMMOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SDDMMOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SDDMMOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dnmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatAOperands(&dnmatARawOperand, 1); ::llvm::SMLoc dnmatAOperandsLoc; (void)dnmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatBOperands(&dnmatBRawOperand, 1); ::llvm::SMLoc dnmatBOperandsLoc; (void)dnmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatCOperands(&spmatCRawOperand, 1); ::llvm::SMLoc spmatCOperandsLoc; (void)spmatCOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bufferRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bufferOperands(&bufferRawOperand, 1); ::llvm::SMLoc bufferOperandsLoc; (void)bufferOperandsLoc; ::mlir::Type bufferRawType{}; ::llvm::ArrayRef<::mlir::Type> bufferTypes(&bufferRawType, 1); ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } dnmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatCRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bufferOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bufferRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); bufferRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatAOperands, odsBuildableType1, dnmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatBOperands, odsBuildableType1, dnmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatCOperands, odsBuildableType2, spmatCOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bufferOperands, bufferTypes, bufferOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SDDMMOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDnmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatC(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBuffer(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBuffer().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SDDMMOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SDDMMOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SetCsrPointersOp definitions //===----------------------------------------------------------------------===// namespace detail { SetCsrPointersOpGenericAdaptorBase::SetCsrPointersOpGenericAdaptorBase(SetCsrPointersOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SetCsrPointersOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SetCsrPointersOpAdaptor::SetCsrPointersOpAdaptor(SetCsrPointersOp op) : SetCsrPointersOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SetCsrPointersOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SetCsrPointersOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SetCsrPointersOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SetCsrPointersOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SetCsrPointersOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SetCsrPointersOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SetCsrPointersOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SetCsrPointersOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SetCsrPointersOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SetCsrPointersOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SetCsrPointersOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SetCsrPointersOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); odsState.addOperands(positions); odsState.addOperands(coordinates); odsState.addOperands(values); if (asyncToken) odsState.addTypes(asyncToken); } SetCsrPointersOp SetCsrPointersOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat), std::forward(positions), std::forward(coordinates), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetCsrPointersOp SetCsrPointersOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat), std::forward(positions), std::forward(coordinates), std::forward(values)); } void SetCsrPointersOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); odsState.addOperands(positions); odsState.addOperands(coordinates); odsState.addOperands(values); odsState.addTypes(resultTypes); } SetCsrPointersOp SetCsrPointersOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat), std::forward(positions), std::forward(coordinates), std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetCsrPointersOp SetCsrPointersOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat, ::mlir::Value positions, ::mlir::Value coordinates, ::mlir::Value values) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat), std::forward(positions), std::forward(coordinates), std::forward(values)); } void SetCsrPointersOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } SetCsrPointersOp SetCsrPointersOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetCsrPointersOp SetCsrPointersOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SetCsrPointersOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SetCsrPointersOp SetCsrPointersOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetCsrPointersOp SetCsrPointersOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SetCsrPointersOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SetCsrPointersOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SetCsrPointersOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatOperands(&spmatRawOperand, 1); ::llvm::SMLoc spmatOperandsLoc; (void)spmatOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand positionsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> positionsOperands(&positionsRawOperand, 1); ::llvm::SMLoc positionsOperandsLoc; (void)positionsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand coordinatesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> coordinatesOperands(&coordinatesRawOperand, 1); ::llvm::SMLoc coordinatesOperandsLoc; (void)coordinatesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valuesRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valuesOperands(&valuesRawOperand, 1); ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::mlir::Type positionsRawType{}; ::llvm::ArrayRef<::mlir::Type> positionsTypes(&positionsRawType, 1); ::mlir::Type coordinatesRawType{}; ::llvm::ArrayRef<::mlir::Type> coordinatesTypes(&coordinatesRawType, 1); ::mlir::Type valuesRawType{}; ::llvm::ArrayRef<::mlir::Type> valuesTypes(&valuesRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); positionsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(positionsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); coordinatesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(coordinatesRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valuesRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); positionsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); coordinatesRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valuesRawType = type; } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatOperands, odsBuildableType1, spmatOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(positionsOperands, positionsTypes, positionsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(coordinatesOperands, coordinatesTypes, coordinatesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SetCsrPointersOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmat(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getPositions(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getCoordinates(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValues(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getPositions().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getCoordinates().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValues().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SetCsrPointersOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SetDefaultDeviceOp definitions //===----------------------------------------------------------------------===// namespace detail { SetDefaultDeviceOpGenericAdaptorBase::SetDefaultDeviceOpGenericAdaptorBase(SetDefaultDeviceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail SetDefaultDeviceOpAdaptor::SetDefaultDeviceOpAdaptor(SetDefaultDeviceOp op) : SetDefaultDeviceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SetDefaultDeviceOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult SetDefaultDeviceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SetDefaultDeviceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SetDefaultDeviceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SetDefaultDeviceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SetDefaultDeviceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SetDefaultDeviceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SetDefaultDeviceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SetDefaultDeviceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value devIndex) { odsState.addOperands(devIndex); } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value devIndex) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(devIndex)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value devIndex) { return create(builder, builder.getLoc(), std::forward(devIndex)); } void SetDefaultDeviceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value devIndex) { odsState.addOperands(devIndex); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value devIndex) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(devIndex)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value devIndex) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(devIndex)); } void SetDefaultDeviceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SetDefaultDeviceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SetDefaultDeviceOp SetDefaultDeviceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SetDefaultDeviceOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SetDefaultDeviceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SetDefaultDeviceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand devIndexRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> devIndexOperands(&devIndexRawOperand, 1); ::llvm::SMLoc devIndexOperandsLoc; (void)devIndexOperandsLoc; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } devIndexOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(devIndexRawOperand)) return ::mlir::failure(); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(32); if (parser.resolveOperands(devIndexOperands, odsBuildableType0, devIndexOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SetDefaultDeviceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' '; _odsPrinter << getDevIndex(); } void SetDefaultDeviceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SetDefaultDeviceOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ShuffleOp definitions //===----------------------------------------------------------------------===// namespace detail { ShuffleOpGenericAdaptorBase::ShuffleOpGenericAdaptorBase(ShuffleOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::ShuffleMode ShuffleOpGenericAdaptorBase::getMode() { auto attr = getModeAttr(); return attr.getValue(); } } // namespace detail ShuffleOpAdaptor::ShuffleOpAdaptor(ShuffleOp op) : ShuffleOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ShuffleOpAdaptor::verify(::mlir::Location loc) { auto tblgen_mode = getProperties().mode; (void)tblgen_mode; if (!tblgen_mode) return emitError(loc, "'gpu.shuffle' op requires attribute 'mode'"); if (tblgen_mode && !((::llvm::isa<::mlir::gpu::ShuffleModeAttr>(tblgen_mode)))) return emitError(loc, "'gpu.shuffle' op attribute 'mode' failed to satisfy constraint: Indexing modes supported by gpu.shuffle."); return ::mlir::success(); } void ShuffleOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "shuffleResult"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "valid"); } ::llvm::LogicalResult ShuffleOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.mode; auto attr = dict.get("mode"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `mode` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ShuffleOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.mode; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("mode", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ShuffleOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.mode.getAsOpaquePointer())); } std::optional ShuffleOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "mode") return prop.mode; return std::nullopt; } void ShuffleOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "mode") { prop.mode = ::llvm::dyn_cast_or_null>(value); return; } } void ShuffleOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.mode) attrs.append("mode", prop.mode); } ::llvm::LogicalResult ShuffleOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getModeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps22(attr, "mode", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ShuffleOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.mode))) return ::mlir::failure(); return ::mlir::success(); } void ShuffleOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.mode); } ::mlir::gpu::ShuffleMode ShuffleOp::getMode() { auto attr = getModeAttr(); return attr.getValue(); } void ShuffleOp::setMode(::mlir::gpu::ShuffleMode attrValue) { getProperties().mode = ::mlir::gpu::ShuffleModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, int32_t offset, int32_t width, ShuffleMode mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, int32_t offset, int32_t width, ShuffleMode mode) { return create(builder, builder.getLoc(), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = mode; odsState.addTypes(shuffleResult); odsState.addTypes(valid); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(shuffleResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { return create(builder, builder.getLoc(), std::forward(shuffleResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = mode; ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ShuffleOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { return create(builder, builder.getLoc(), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = mode; assert(resultTypes.size() == 2u && "mismatched number of results"); odsState.addTypes(resultTypes); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleModeAttr mode) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = ::mlir::gpu::ShuffleModeAttr::get(odsBuilder.getContext(), mode); odsState.addTypes(shuffleResult); odsState.addTypes(valid); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(shuffleResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type shuffleResult, ::mlir::Type valid, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { return create(builder, builder.getLoc(), std::forward(shuffleResult), std::forward(valid), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = ::mlir::gpu::ShuffleModeAttr::get(odsBuilder.getContext(), mode); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ShuffleOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { return create(builder, builder.getLoc(), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { odsState.addOperands(value); odsState.addOperands(offset); odsState.addOperands(width); odsState.getOrAddProperties().mode = ::mlir::gpu::ShuffleModeAttr::get(odsBuilder.getContext(), mode); assert(resultTypes.size() == 2u && "mismatched number of results"); odsState.addTypes(resultTypes); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value offset, ::mlir::Value width, ::mlir::gpu::ShuffleMode mode) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(offset), std::forward(width), std::forward(mode)); } void ShuffleOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ShuffleOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ShuffleOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ShuffleOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ShuffleOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 2u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ShuffleOp ShuffleOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ShuffleOp ShuffleOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ShuffleOp::verifyInvariantsImpl() { auto tblgen_mode = getProperties().mode; (void)tblgen_mode; if (!tblgen_mode) return emitOpError("requires attribute 'mode'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps22(*this, tblgen_mode, "mode"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {value, shuffleResult} have same type"); return ::mlir::success(); } ::llvm::LogicalResult ShuffleOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ShuffleOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(2); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); ::mlir::Type odsInferredType1 = odsBuilder.getIntegerType(1); inferredReturnTypes[0] = odsInferredType0; inferredReturnTypes[1] = odsInferredType1; return ::mlir::success(); } ::mlir::ParseResult ShuffleOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::ShuffleModeAttr modeAttr; ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand offsetRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetOperands(&offsetRawOperand, 1); ::llvm::SMLoc offsetOperandsLoc; (void)offsetOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand widthRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> widthOperands(&widthRawOperand, 1); ::llvm::SMLoc widthOperandsLoc; (void)widthOperandsLoc; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); if (parser.parseCustomAttributeWithFallback(modeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAttr) result.getOrAddProperties().mode = modeAttr; valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); offsetOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(offsetRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); widthOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(widthRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(1); ::mlir::Type odsBuildableType1 = parser.getBuilder().getIntegerType(32); result.addTypes(valueTypes[0]); result.addTypes(odsBuildableType0); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(offsetOperands, odsBuildableType1, offsetOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(widthOperands, odsBuildableType1, widthOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void ShuffleOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getModeAttr()); _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getOffset(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getWidth(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("mode"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void ShuffleOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ShuffleOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ShuffleOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpGEMMCopyOp definitions //===----------------------------------------------------------------------===// namespace detail { SpGEMMCopyOpGenericAdaptorBase::SpGEMMCopyOpGenericAdaptorBase(SpGEMMCopyOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpGEMMCopyOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SpGEMMCopyOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpGEMMCopyOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SpGEMMCopyOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SpGEMMCopyOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpGEMMCopyOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SpGEMMCopyOpAdaptor::SpGEMMCopyOpAdaptor(SpGEMMCopyOp op) : SpGEMMCopyOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpGEMMCopyOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spgemm_copy' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spgemm_copy' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.spgemm_copy' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spgemm_copy' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } std::pair SpGEMMCopyOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpGEMMCopyOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpGEMMCopyOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpGEMMCopyOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SpGEMMCopyOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpGEMMCopyOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer())); } std::optional SpGEMMCopyOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; return std::nullopt; } void SpGEMMCopyOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } } void SpGEMMCopyOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); } ::llvm::LogicalResult SpGEMMCopyOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMCopyOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMCopyOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); } ::mlir::gpu::TransposeMode SpGEMMCopyOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SpGEMMCopyOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpGEMMCopyOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SpGEMMCopyOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpGEMMCopyOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpGEMMCopyOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpGEMMCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, asyncToken, asyncDependencies, desc, modeA, modeB, spmatA, spmatB, spmatC, computeType); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(resultTypes); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(resultTypes); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpGEMMCopyOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCopyOp SpGEMMCopyOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpGEMMCopyOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpGEMMCopyOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps20(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMCopyOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpGEMMCopyOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatBOperands(&spmatBRawOperand, 1); ::llvm::SMLoc spmatBOperandsLoc; (void)spmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatCOperands(&spmatCRawOperand, 1); ::llvm::SMLoc spmatCOperandsLoc; (void)spmatCOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand descRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descOperands(&descRawOperand, 1); ::llvm::SMLoc descOperandsLoc; (void)descOperandsLoc; ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatCRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); descOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(descRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpGEMMOpHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(descOperands, odsBuildableType1, descOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType2, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatBOperands, odsBuildableType2, spmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatCOperands, odsBuildableType2, spmatCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMCopyOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatC(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDesc(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpGEMMCopyOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpGEMMCopyOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpGEMMCreateDescrOp definitions //===----------------------------------------------------------------------===// namespace detail { SpGEMMCreateDescrOpGenericAdaptorBase::SpGEMMCreateDescrOpGenericAdaptorBase(SpGEMMCreateDescrOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpGEMMCreateDescrOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SpGEMMCreateDescrOpAdaptor::SpGEMMCreateDescrOpAdaptor(SpGEMMCreateDescrOp op) : SpGEMMCreateDescrOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpGEMMCreateDescrOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void SpGEMMCreateDescrOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "desc"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair SpGEMMCreateDescrOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpGEMMCreateDescrOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpGEMMCreateDescrOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpGEMMCreateDescrOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SpGEMMCreateDescrOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpGEMMCreateDescrOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SpGEMMCreateDescrOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SpGEMMCreateDescrOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SpGEMMCreateDescrOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SpGEMMCreateDescrOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SpGEMMCreateDescrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type desc, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { odsState.addOperands(asyncDependencies); odsState.addTypes(desc); if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type desc, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(desc), std::forward(asyncToken), std::forward(asyncDependencies)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type desc, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { return create(builder, builder.getLoc(), std::forward(desc), std::forward(asyncToken), std::forward(asyncDependencies)); } void SpGEMMCreateDescrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpGEMMCreateDescrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMCreateDescrOp SpGEMMCreateDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SpGEMMCreateDescrOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps20(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMCreateDescrOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpGEMMCreateDescrOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::SparseSpGEMMOpHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMCreateDescrOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpGEMMCreateDescrOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpGEMMDestroyDescrOp definitions //===----------------------------------------------------------------------===// namespace detail { SpGEMMDestroyDescrOpGenericAdaptorBase::SpGEMMDestroyDescrOpGenericAdaptorBase(SpGEMMDestroyDescrOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpGEMMDestroyDescrOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SpGEMMDestroyDescrOpAdaptor::SpGEMMDestroyDescrOpAdaptor(SpGEMMDestroyDescrOp op) : SpGEMMDestroyDescrOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpGEMMDestroyDescrOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SpGEMMDestroyDescrOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpGEMMDestroyDescrOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpGEMMDestroyDescrOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpGEMMDestroyDescrOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SpGEMMDestroyDescrOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpGEMMDestroyDescrOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SpGEMMDestroyDescrOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SpGEMMDestroyDescrOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SpGEMMDestroyDescrOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SpGEMMDestroyDescrOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SpGEMMDestroyDescrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc)); } void SpGEMMDestroyDescrOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addTypes(resultTypes); } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc)); } void SpGEMMDestroyDescrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpGEMMDestroyDescrOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMDestroyDescrOp SpGEMMDestroyDescrOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SpGEMMDestroyDescrOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps20(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMDestroyDescrOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpGEMMDestroyDescrOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand descRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descOperands(&descRawOperand, 1); ::llvm::SMLoc descOperandsLoc; (void)descOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } descOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(descRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpGEMMOpHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(descOperands, odsBuildableType1, descOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMDestroyDescrOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getDesc(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpGEMMDestroyDescrOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpGEMMWorkEstimationOrComputeOp definitions //===----------------------------------------------------------------------===// namespace detail { SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase(SpGEMMWorkEstimationOrComputeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::getKind() { auto attr = getKindAttr(); return attr.getValue(); } } // namespace detail SpGEMMWorkEstimationOrComputeOpAdaptor::SpGEMMWorkEstimationOrComputeOpAdaptor(SpGEMMWorkEstimationOrComputeOp op) : SpGEMMWorkEstimationOrComputeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op requires attribute 'computeType'"); auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (!tblgen_kind) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op requires attribute 'kind'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op attribute 'computeType' failed to satisfy constraint: any type attribute"); if (tblgen_kind && !((::llvm::isa<::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr>(tblgen_kind)))) return emitError(loc, "'gpu.spgemm_work_estimation_or_compute' op attribute 'kind' failed to satisfy constraint: choose whether spgemm_work_estimation_or_compute does work estimation or compute"); return ::mlir::success(); } void SpGEMMWorkEstimationOrComputeOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "bufferSzNew"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair SpGEMMWorkEstimationOrComputeOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 6) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpGEMMWorkEstimationOrComputeOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpGEMMWorkEstimationOrComputeOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.kind; auto attr = dict.get("kind"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kind` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SpGEMMWorkEstimationOrComputeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.kind; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("kind", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpGEMMWorkEstimationOrComputeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.kind.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer())); } std::optional SpGEMMWorkEstimationOrComputeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "kind") return prop.kind; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; return std::nullopt; } void SpGEMMWorkEstimationOrComputeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "kind") { prop.kind = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } } void SpGEMMWorkEstimationOrComputeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.kind) attrs.append("kind", prop.kind); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); } ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getKindAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps23(attr, "kind", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.kind))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMWorkEstimationOrComputeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeAttribute(prop.kind); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); } ::mlir::gpu::TransposeMode SpGEMMWorkEstimationOrComputeOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SpGEMMWorkEstimationOrComputeOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpGEMMWorkEstimationOrComputeOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind SpGEMMWorkEstimationOrComputeOp::getKind() { auto attr = getKindAttr(); return attr.getValue(); } void SpGEMMWorkEstimationOrComputeOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpGEMMWorkEstimationOrComputeOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpGEMMWorkEstimationOrComputeOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpGEMMWorkEstimationOrComputeOp::setKind(::mlir::gpu::SpGEMMWorkEstimationOrComputeKind attrValue) { getProperties().kind = ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type bufferSzNew, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType, Value bufferSz, Value buffer) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; auto kind = gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION; return build(odsBuilder, odsState, bufferSzNew, asyncToken, asyncDependencies, desc, modeA, modeB, spmatA, spmatB, spmatC, computeType, bufferSz, buffer, kind); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type bufferSzNew, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType, Value bufferSz, Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, Type bufferSzNew, Type asyncToken, ValueRange asyncDependencies, Value desc, Value spmatA, Value spmatB, Value spmatC, Type computeType, Value bufferSz, Value buffer) { return create(builder, builder.getLoc(), std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.addOperands(bufferSz); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.getOrAddProperties().kind = kind; odsState.addTypes(bufferSzNew); if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { return create(builder, builder.getLoc(), std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.addOperands(bufferSz); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.getOrAddProperties().kind = kind; assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::TypeAttr computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kind) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.addOperands(bufferSz); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.getOrAddProperties().kind = ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr::get(odsBuilder.getContext(), kind); odsState.addTypes(bufferSzNew); if (asyncToken) odsState.addTypes(asyncToken); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSzNew, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { return create(builder, builder.getLoc(), std::forward(bufferSzNew), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { odsState.addOperands(asyncDependencies); odsState.addOperands(desc); odsState.addOperands(spmatA); odsState.addOperands(spmatB); odsState.addOperands(spmatC); odsState.addOperands(bufferSz); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.getOrAddProperties().kind = ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr::get(odsBuilder.getContext(), kind); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value desc, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value spmatB, ::mlir::Value spmatC, ::mlir::Type computeType, ::mlir::Value bufferSz, ::mlir::Value buffer, ::mlir::gpu::SpGEMMWorkEstimationOrComputeKind kind) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(desc), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(spmatB), std::forward(spmatC), std::forward(computeType), std::forward(bufferSz), std::forward(buffer), std::forward(kind)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpGEMMWorkEstimationOrComputeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 6u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpGEMMWorkEstimationOrComputeOp SpGEMMWorkEstimationOrComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpGEMMWorkEstimationOrComputeOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (!tblgen_kind) return emitOpError("requires attribute 'kind'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps23(*this, tblgen_kind, "kind"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps20(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup5 = getODSOperands(5); for (auto v : valueGroup5) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup6 = getODSOperands(6); for (auto v : valueGroup6) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpGEMMWorkEstimationOrComputeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpGEMMWorkEstimationOrComputeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::gpu::SpGEMMWorkEstimationOrComputeKindAttr kindAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatBOperands(&spmatBRawOperand, 1); ::llvm::SMLoc spmatBOperandsLoc; (void)spmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand spmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatCOperands(&spmatCRawOperand, 1); ::llvm::SMLoc spmatCOperandsLoc; (void)spmatCOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand descRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> descOperands(&descRawOperand, 1); ::llvm::SMLoc descOperandsLoc; (void)descOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bufferSzRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bufferSzOperands(&bufferSzRawOperand, 1); ::llvm::SMLoc bufferSzOperandsLoc; (void)bufferSzOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bufferRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bufferOperands(&bufferRawOperand, 1); ::llvm::SMLoc bufferOperandsLoc; (void)bufferOperandsLoc; ::mlir::TypeAttr computeTypeAttr; ::mlir::Type bufferRawType{}; ::llvm::ArrayRef<::mlir::Type> bufferTypes(&bufferRawType, 1); { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } if (parser.parseLBrace()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(kindAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (kindAttr) result.getOrAddProperties().kind = kindAttr; if (parser.parseRBrace()) return ::mlir::failure(); spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); spmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatCRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); descOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(descRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bufferSzOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bufferSzRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bufferOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bufferRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); bufferRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseSpGEMMOpHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType3 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(descOperands, odsBuildableType2, descOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType3, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatBOperands, odsBuildableType3, spmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatCOperands, odsBuildableType3, spmatCOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bufferSzOperands, odsBuildableType0, bufferSzOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bufferOperands, bufferTypes, bufferOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpGEMMWorkEstimationOrComputeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getKindAttr()); _odsPrinter << "}"; _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getSpmatC(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDesc(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBufferSz(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBuffer(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("kind"); elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getBuffer().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpGEMMWorkEstimationOrComputeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpGEMMWorkEstimationOrComputeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpMMBufferSizeOp definitions //===----------------------------------------------------------------------===// namespace detail { SpMMBufferSizeOpGenericAdaptorBase::SpMMBufferSizeOpGenericAdaptorBase(SpMMBufferSizeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpMMBufferSizeOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SpMMBufferSizeOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpMMBufferSizeOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SpMMBufferSizeOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SpMMBufferSizeOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpMMBufferSizeOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SpMMBufferSizeOpAdaptor::SpMMBufferSizeOpAdaptor(SpMMBufferSizeOp op) : SpMMBufferSizeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpMMBufferSizeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spmm_buffer_size' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spmm_buffer_size' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.spmm_buffer_size' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spmm_buffer_size' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } void SpMMBufferSizeOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "bufferSzs"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair SpMMBufferSizeOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpMMBufferSizeOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpMMBufferSizeOp::getODSResultIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().resultSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::llvm::LogicalResult SpMMBufferSizeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("resultSegmentSizes"); if (!attr) attr = dict.get("result_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.resultSegmentSizes, attr, [&]() { return emitError() << "for `resultSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SpMMBufferSizeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } { const auto &propStorage = prop.resultSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("resultSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpMMBufferSizeOp::computePropertiesHash(const Properties &prop) { auto hash_resultSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer()), hash_resultSegmentSizes_(prop.resultSegmentSizes)); } std::optional SpMMBufferSizeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; if (name == "result_segment_sizes" || name == "resultSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.resultSegmentSizes); }(); return std::nullopt; } void SpMMBufferSizeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } if (name == "result_segment_sizes" || name == "resultSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.resultSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.resultSegmentSizes.begin()); return; } } void SpMMBufferSizeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); attrs.append("resultSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.resultSegmentSizes); }()); } ::llvm::LogicalResult SpMMBufferSizeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpMMBufferSizeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.resultSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.resultSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SpMMBufferSizeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.resultSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.resultSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::gpu::TransposeMode SpMMBufferSizeOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SpMMBufferSizeOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpMMBufferSizeOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SpMMBufferSizeOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMMBufferSizeOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMMBufferSizeOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type bufferSzs, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, bufferSzs, asyncToken, asyncDependencies, modeA, modeB, spmatA, dnmatB, dnmatC, computeType); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type bufferSzs, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, Type bufferSzs, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(bufferSzs); if (asyncToken) odsState.addTypes(asyncToken); ::llvm::copy(::llvm::ArrayRef({static_cast(bufferSzs.size()), (asyncToken ? 1 : 0)}), odsState.getOrAddProperties().resultSegmentSizes.begin()); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(resultTypes); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(bufferSzs); if (asyncToken) odsState.addTypes(asyncToken); ::llvm::copy(::llvm::ArrayRef({static_cast(bufferSzs.size()), (asyncToken ? 1 : 0)}), odsState.getOrAddProperties().resultSegmentSizes.begin()); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange bufferSzs, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSzs), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(resultTypes); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpMMBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMBufferSizeOp SpMMBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpMMBufferSizeOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpMMBufferSizeOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpMMBufferSizeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpMMBufferSizeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatBOperands(&dnmatBRawOperand, 1); ::llvm::SMLoc dnmatBOperandsLoc; (void)dnmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatCOperands(&dnmatCRawOperand, 1); ::llvm::SMLoc dnmatCOperandsLoc; (void)dnmatCOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> bufferSzsTypes; ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatCRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(bufferSzsTypes)) return ::mlir::failure(); if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::llvm::copy(::llvm::ArrayRef({static_cast(bufferSzsTypes.size()), static_cast(asyncTokenTypes.size())}), result.getOrAddProperties().resultSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); result.addTypes(bufferSzsTypes); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType1, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatBOperands, odsBuildableType2, dnmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatCOperands, odsBuildableType2, dnmatCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpMMBufferSizeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatC(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("resultSegmentSizes"); elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getBufferSzs().getTypes(); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpMMBufferSizeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpMMBufferSizeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpMMOp definitions //===----------------------------------------------------------------------===// namespace detail { SpMMOpGenericAdaptorBase::SpMMOpGenericAdaptorBase(SpMMOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpMMOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::gpu::TransposeModeAttr SpMMOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpMMOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeModeAttr SpMMOpGenericAdaptorBase::getModeBAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeB); return attr; } ::mlir::gpu::TransposeMode SpMMOpGenericAdaptorBase::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpMMOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SpMMOpAdaptor::SpMMOpAdaptor(SpMMOp op) : SpMMOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpMMOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spmm' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spmm' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_modeB && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeB)))) return emitError(loc, "'gpu.spmm' op attribute 'modeB' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spmm' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } std::pair SpMMOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SpMMOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(0u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SpMMOp::getBuffersMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } std::pair SpMMOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpMMOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeB; auto attr = dict.get("modeB"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeB` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SpMMOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } { const auto &propStorage = prop.modeB; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeB", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpMMOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer()), llvm::hash_value(prop.modeB.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SpMMOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; if (name == "modeB") return prop.modeB; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SpMMOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeB") { prop.modeB = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SpMMOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); if (prop.modeB) attrs.append("modeB", prop.modeB); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SpMMOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeB", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpMMOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeB))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SpMMOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); writer.writeOptionalAttribute(prop.modeB); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::gpu::TransposeMode SpMMOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::gpu::TransposeMode SpMMOp::getModeB() { auto attr = getModeBAttr(); return attr.getValue(); } ::mlir::Type SpMMOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SpMMOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMMOp::setModeB(::mlir::gpu::TransposeMode attrValue) { getProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMMOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType, ValueRange buffers) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; auto modeB = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, asyncToken, asyncDependencies, modeA, modeB, spmatA, dnmatB, dnmatC, computeType, buffers); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType, ValueRange buffers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnmatB, Value dnmatC, Type computeType, ValueRange buffers) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); } void SpMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.addOperands(buffers); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, 1, 1, static_cast(buffers.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; if (asyncToken) odsState.addTypes(asyncToken); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); } void SpMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.addOperands(buffers); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, 1, 1, static_cast(buffers.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } if (modeB) { odsState.getOrAddProperties().modeB = modeB; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(resultTypes); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::gpu::TransposeModeAttr modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::TypeAttr computeType, ::mlir::ValueRange buffers) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); } void SpMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.addOperands(buffers); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, 1, 1, static_cast(buffers.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); if (asyncToken) odsState.addTypes(asyncToken); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); } void SpMMOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnmatB); odsState.addOperands(dnmatC); odsState.addOperands(buffers); ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependencies.size()), 1, 1, 1, static_cast(buffers.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeB); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(resultTypes); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::gpu::TransposeMode modeB, ::mlir::Value spmatA, ::mlir::Value dnmatB, ::mlir::Value dnmatC, ::mlir::Type computeType, ::mlir::ValueRange buffers) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(modeB), std::forward(spmatA), std::forward(dnmatB), std::forward(dnmatC), std::forward(computeType), std::forward(buffers)); } void SpMMOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpMMOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SpMMOp SpMMOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMMOp SpMMOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpMMOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); if (!properties.modeB) properties.modeB = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpMMOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; auto tblgen_modeB = getProperties().modeB; (void)tblgen_modeB; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeB, "modeB"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps21(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpMMOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpMMOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatBOperands(&dnmatBRawOperand, 1); ::llvm::SMLoc dnmatBOperandsLoc; (void)dnmatBOperandsLoc; ::mlir::gpu::TransposeModeAttr modeBAttr; ::mlir::OpAsmParser::UnresolvedOperand dnmatCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnmatCOperands(&dnmatCRawOperand, 1); ::llvm::SMLoc dnmatCOperandsLoc; (void)dnmatCOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> buffersOperands; ::llvm::SMLoc buffersOperandsLoc; (void)buffersOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> buffersTypes; ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatBRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeBAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeBAttr) result.getOrAddProperties().modeB = modeBAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnmatCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnmatCRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); buffersOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(buffersOperands)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(buffersTypes)) return ::mlir::failure(); if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::llvm::copy(::llvm::ArrayRef({static_cast(asyncDependenciesOperands.size()), 1, 1, 1, static_cast(buffersOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType1, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatBOperands, odsBuildableType2, dnmatBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnmatCOperands, odsBuildableType2, dnmatCOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(buffersOperands, buffersTypes, buffersOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpMMOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatB(); if (getModeBAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeBAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnmatC(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBuffers(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("modeA"); elidedAttrs.push_back("modeB"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeBAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeB"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getBuffers().getTypes(); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpMMOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpMMOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpMVBufferSizeOp definitions //===----------------------------------------------------------------------===// namespace detail { SpMVBufferSizeOpGenericAdaptorBase::SpMVBufferSizeOpGenericAdaptorBase(SpMVBufferSizeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpMVBufferSizeOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SpMVBufferSizeOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpMVBufferSizeOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::Type SpMVBufferSizeOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SpMVBufferSizeOpAdaptor::SpMVBufferSizeOpAdaptor(SpMVBufferSizeOp op) : SpMVBufferSizeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpMVBufferSizeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spmv_buffer_size' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spmv_buffer_size' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spmv_buffer_size' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } void SpMVBufferSizeOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "bufferSz"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "asyncToken"); } std::pair SpMVBufferSizeOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpMVBufferSizeOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpMVBufferSizeOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpMVBufferSizeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SpMVBufferSizeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpMVBufferSizeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer())); } std::optional SpMVBufferSizeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; return std::nullopt; } void SpMVBufferSizeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } } void SpMVBufferSizeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); } ::llvm::LogicalResult SpMVBufferSizeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpMVBufferSizeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); return ::mlir::success(); } void SpMVBufferSizeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); } ::mlir::gpu::TransposeMode SpMVBufferSizeOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::Type SpMVBufferSizeOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SpMVBufferSizeOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMVBufferSizeOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, bufferSz, asyncToken, asyncDependencies, modeA, spmatA, dnX, dnY, computeType); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, Type bufferSz, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(bufferSz); if (asyncToken) odsState.addTypes(asyncToken); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } odsState.getOrAddProperties().computeType = computeType; assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(bufferSz); if (asyncToken) odsState.addTypes(asyncToken); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type bufferSz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(bufferSz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); assert(resultTypes.size() >= 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpMVBufferSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVBufferSizeOp SpMVBufferSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpMVBufferSizeOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpMVBufferSizeOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); if (valueGroup1.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpMVBufferSizeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpMVBufferSizeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnXRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnXOperands(&dnXRawOperand, 1); ::llvm::SMLoc dnXOperandsLoc; (void)dnXOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dnYRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnYOperands(&dnYRawOperand, 1); ::llvm::SMLoc dnYOperandsLoc; (void)dnYOperandsLoc; ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnXOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnXRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dnYOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnYRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType3 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType2, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnXOperands, odsBuildableType3, dnXOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnYOperands, odsBuildableType3, dnYOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpMVBufferSizeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnX(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnY(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("modeA"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpMVBufferSizeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpMVBufferSizeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpMVOp definitions //===----------------------------------------------------------------------===// namespace detail { SpMVOpGenericAdaptorBase::SpMVOpGenericAdaptorBase(SpMVOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpMVOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::TransposeModeAttr SpMVOpGenericAdaptorBase::getModeAAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::gpu::TransposeModeAttr>(getProperties().modeA); return attr; } ::mlir::gpu::TransposeMode SpMVOpGenericAdaptorBase::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::Type SpMVOpGenericAdaptorBase::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } } // namespace detail SpMVOpAdaptor::SpMVOpAdaptor(SpMVOp op) : SpMVOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpMVOpAdaptor::verify(::mlir::Location loc) { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitError(loc, "'gpu.spmv' op requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; if (tblgen_modeA && !((::llvm::isa<::mlir::gpu::TransposeModeAttr>(tblgen_modeA)))) return emitError(loc, "'gpu.spmv' op attribute 'modeA' failed to satisfy constraint: transpose mode of sparse matrix supported by sparse tensor ops"); if (tblgen_computeType && !(((::llvm::isa<::mlir::TypeAttr>(tblgen_computeType))) && ((::llvm::isa<::mlir::Type>(::llvm::cast<::mlir::TypeAttr>(tblgen_computeType).getValue()))) && ((true)))) return emitError(loc, "'gpu.spmv' op attribute 'computeType' failed to satisfy constraint: any type attribute"); return ::mlir::success(); } std::pair SpMVOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false, false, false, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpMVOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpMVOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpMVOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.computeType; auto attr = dict.get("computeType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `computeType` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.modeA; auto attr = dict.get("modeA"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `modeA` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SpMVOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.computeType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("computeType", propStorage)); } { const auto &propStorage = prop.modeA; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("modeA", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpMVOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.computeType.getAsOpaquePointer()), llvm::hash_value(prop.modeA.getAsOpaquePointer())); } std::optional SpMVOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "computeType") return prop.computeType; if (name == "modeA") return prop.modeA; return std::nullopt; } void SpMVOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "computeType") { prop.computeType = ::llvm::dyn_cast_or_null>(value); return; } if (name == "modeA") { prop.modeA = ::llvm::dyn_cast_or_null>(value); return; } } void SpMVOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.computeType) attrs.append("computeType", prop.computeType); if (prop.modeA) attrs.append("modeA", prop.modeA); } ::llvm::LogicalResult SpMVOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getComputeTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(attr, "computeType", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getModeAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(attr, "modeA", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SpMVOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.computeType))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.modeA))) return ::mlir::failure(); return ::mlir::success(); } void SpMVOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.computeType); writer.writeOptionalAttribute(prop.modeA); } ::mlir::gpu::TransposeMode SpMVOp::getModeA() { auto attr = getModeAAttr(); return attr.getValue(); } ::mlir::Type SpMVOp::getComputeType() { auto attr = getComputeTypeAttr(); return ::llvm::cast<::mlir::Type>(attr.getValue()); } void SpMVOp::setModeA(::mlir::gpu::TransposeMode attrValue) { getProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SpMVOp::setComputeType(::mlir::Type attrValue) { getProperties().computeType = ::mlir::TypeAttr::get(attrValue); } void SpMVOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType, Value buffer) { auto modeA = gpu::TransposeMode::NON_TRANSPOSE; return build(odsBuilder, odsState, asyncToken, asyncDependencies, modeA, spmatA, dnX, dnY, computeType, buffer); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType, Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, Type asyncToken, ValueRange asyncDependencies, Value spmatA, Value dnX, Value dnY, Type computeType, Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); } void SpMVOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } odsState.getOrAddProperties().computeType = computeType; if (asyncToken) odsState.addTypes(asyncToken); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); } void SpMVOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.addOperands(buffer); if (modeA) { odsState.getOrAddProperties().modeA = modeA; } odsState.getOrAddProperties().computeType = computeType; odsState.addTypes(resultTypes); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeModeAttr modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::TypeAttr computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); } void SpMVOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); if (asyncToken) odsState.addTypes(asyncToken); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); } void SpMVOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmatA); odsState.addOperands(dnX); odsState.addOperands(dnY); odsState.addOperands(buffer); odsState.getOrAddProperties().modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), modeA); odsState.getOrAddProperties().computeType = ::mlir::TypeAttr::get(computeType); odsState.addTypes(resultTypes); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::gpu::TransposeMode modeA, ::mlir::Value spmatA, ::mlir::Value dnX, ::mlir::Value dnY, ::mlir::Type computeType, ::mlir::Value buffer) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(modeA), std::forward(spmatA), std::forward(dnX), std::forward(dnY), std::forward(computeType), std::forward(buffer)); } void SpMVOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpMVOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } SpMVOp SpMVOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMVOp SpMVOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SpMVOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.modeA) properties.modeA = ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE); } ::llvm::LogicalResult SpMVOp::verifyInvariantsImpl() { auto tblgen_computeType = getProperties().computeType; (void)tblgen_computeType; if (!tblgen_computeType) return emitOpError("requires attribute 'computeType'"); auto tblgen_modeA = getProperties().modeA; (void)tblgen_modeA; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps20(*this, tblgen_modeA, "modeA"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps21(*this, tblgen_computeType, "computeType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpMVOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpMVOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatAOperands(&spmatARawOperand, 1); ::llvm::SMLoc spmatAOperandsLoc; (void)spmatAOperandsLoc; ::mlir::gpu::TransposeModeAttr modeAAttr; ::mlir::OpAsmParser::UnresolvedOperand dnXRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnXOperands(&dnXRawOperand, 1); ::llvm::SMLoc dnXOperandsLoc; (void)dnXOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dnYRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dnYOperands(&dnYRawOperand, 1); ::llvm::SMLoc dnYOperandsLoc; (void)dnYOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bufferRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bufferOperands(&bufferRawOperand, 1); ::llvm::SMLoc bufferOperandsLoc; (void)bufferOperandsLoc; ::mlir::Type bufferRawType{}; ::llvm::ArrayRef<::mlir::Type> bufferTypes(&bufferRawType, 1); ::mlir::TypeAttr computeTypeAttr; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatARawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalLBrace())) { if (parser.parseCustomAttributeWithFallback(modeAAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (modeAAttr) result.getOrAddProperties().modeA = modeAAttr; if (parser.parseRBrace()) return ::mlir::failure(); } if (parser.parseComma()) return ::mlir::failure(); dnXOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnXRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dnYOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dnYRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bufferOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bufferRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); bufferRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(computeTypeAttr, parser.getBuilder().getType<::mlir::NoneType>())) { return ::mlir::failure(); } if (computeTypeAttr) result.getOrAddProperties().computeType = computeTypeAttr; ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType1 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseDnTensorHandleType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatAOperands, odsBuildableType1, spmatAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnXOperands, odsBuildableType2, dnXOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dnYOperands, odsBuildableType2, dnYOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(bufferOperands, bufferTypes, bufferOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpMVOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmatA(); if (getModeAAttr() != ::mlir::gpu::TransposeModeAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TransposeMode::NON_TRANSPOSE)) { _odsPrinter << "{"; _odsPrinter.printStrippedAttrOrType(getModeAAttr()); _odsPrinter << "}"; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnX(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDnY(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBuffer(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("modeA"); elidedAttrs.push_back("computeType"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getModeAAttr(); if(attr && (attr == ::mlir::gpu::TransposeModeAttr::get(odsBuilder.getContext(), TransposeMode::NON_TRANSPOSE))) elidedAttrs.push_back("modeA"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBuffer().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getComputeTypeAttr()); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpMVOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SpMVOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SpMatGetSizeOp definitions //===----------------------------------------------------------------------===// namespace detail { SpMatGetSizeOpGenericAdaptorBase::SpMatGetSizeOpGenericAdaptorBase(SpMatGetSizeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SpMatGetSizeOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SpMatGetSizeOpAdaptor::SpMatGetSizeOpAdaptor(SpMatGetSizeOp op) : SpMatGetSizeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SpMatGetSizeOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } void SpMatGetSizeOp::getAsmResultNames(::mlir::OpAsmSetValueNameFn setNameFn) { auto resultGroup0 = getODSResults(0); if (!resultGroup0.empty()) setNameFn(*resultGroup0.begin(), "rows"); auto resultGroup1 = getODSResults(1); if (!resultGroup1.empty()) setNameFn(*resultGroup1.begin(), "cols"); auto resultGroup2 = getODSResults(2); if (!resultGroup2.empty()) setNameFn(*resultGroup2.begin(), "nnz"); auto resultGroup3 = getODSResults(3); if (!resultGroup3.empty()) setNameFn(*resultGroup3.begin(), "asyncToken"); } std::pair SpMatGetSizeOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SpMatGetSizeOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair SpMatGetSizeOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 3) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult SpMatGetSizeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SpMatGetSizeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SpMatGetSizeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SpMatGetSizeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SpMatGetSizeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SpMatGetSizeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SpMatGetSizeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SpMatGetSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type rows, ::mlir::Type cols, ::mlir::Type nnz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); odsState.addTypes(rows); odsState.addTypes(cols); odsState.addTypes(nnz); if (asyncToken) odsState.addTypes(asyncToken); } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type rows, ::mlir::Type cols, ::mlir::Type nnz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type rows, ::mlir::Type cols, ::mlir::Type nnz, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { return create(builder, builder.getLoc(), std::forward(rows), std::forward(cols), std::forward(nnz), std::forward(asyncToken), std::forward(asyncDependencies), std::forward(spmat)); } void SpMatGetSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { odsState.addOperands(asyncDependencies); odsState.addOperands(spmat); assert(resultTypes.size() >= 3u && "mismatched number of results"); odsState.addTypes(resultTypes); } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange asyncDependencies, ::mlir::Value spmat) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(asyncDependencies), std::forward(spmat)); } void SpMatGetSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() >= 3u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SpMatGetSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() >= 3u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SpMatGetSizeOp SpMatGetSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SpMatGetSizeOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSResults(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSResults(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSResults(3); if (valueGroup3.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SpMatGetSizeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SpMatGetSizeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand spmatRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> spmatOperands(&spmatRawOperand, 1); ::llvm::SMLoc spmatOperandsLoc; (void)spmatOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } spmatOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(spmatRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); ::mlir::Type odsBuildableType1 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); ::mlir::Type odsBuildableType2 = mlir::gpu::SparseSpMatHandleType::get(parser.getBuilder().getContext()); result.addTypes(odsBuildableType0); result.addTypes(odsBuildableType0); result.addTypes(odsBuildableType0); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType1, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(spmatOperands, odsBuildableType2, spmatOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SpMatGetSizeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); _odsPrinter << ' '; _odsPrinter << getSpmat(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SpMatGetSizeOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupBroadcastOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupBroadcastOpGenericAdaptorBase::SubgroupBroadcastOpGenericAdaptorBase(SubgroupBroadcastOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupBroadcastOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::BroadcastType SubgroupBroadcastOpGenericAdaptorBase::getBroadcastType() { auto attr = getBroadcastTypeAttr(); return attr.getValue(); } } // namespace detail SubgroupBroadcastOpAdaptor::SubgroupBroadcastOpAdaptor(SubgroupBroadcastOp op) : SubgroupBroadcastOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupBroadcastOpAdaptor::verify(::mlir::Location loc) { auto tblgen_broadcast_type = getProperties().broadcast_type; (void)tblgen_broadcast_type; if (!tblgen_broadcast_type) return emitError(loc, "'gpu.subgroup_broadcast' op requires attribute 'broadcast_type'"); if (tblgen_broadcast_type && !((::llvm::isa<::mlir::gpu::BroadcastTypeAttr>(tblgen_broadcast_type)))) return emitError(loc, "'gpu.subgroup_broadcast' op attribute 'broadcast_type' failed to satisfy constraint: a lane to broadcast from"); return ::mlir::success(); } std::pair SubgroupBroadcastOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupBroadcastOp::getLaneMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupBroadcastOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.broadcast_type; auto attr = dict.get("broadcast_type"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `broadcast_type` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupBroadcastOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.broadcast_type; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("broadcast_type", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupBroadcastOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.broadcast_type.getAsOpaquePointer())); } std::optional SubgroupBroadcastOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "broadcast_type") return prop.broadcast_type; return std::nullopt; } void SubgroupBroadcastOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "broadcast_type") { prop.broadcast_type = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupBroadcastOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.broadcast_type) attrs.append("broadcast_type", prop.broadcast_type); } ::llvm::LogicalResult SubgroupBroadcastOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getBroadcastTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps24(attr, "broadcast_type", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupBroadcastOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.broadcast_type))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupBroadcastOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.broadcast_type); } ::mlir::gpu::BroadcastType SubgroupBroadcastOp::getBroadcastType() { auto attr = getBroadcastTypeAttr(); return attr.getValue(); } void SubgroupBroadcastOp::setBroadcastType(::mlir::gpu::BroadcastType attrValue) { getProperties().broadcast_type = ::mlir::gpu::BroadcastTypeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = broadcast_type; odsState.addTypes(result); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { return create(builder, builder.getLoc(), std::forward(result), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = broadcast_type; ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupBroadcastOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { return create(builder, builder.getLoc(), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = broadcast_type; assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastTypeAttr broadcast_type) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = ::mlir::gpu::BroadcastTypeAttr::get(odsBuilder.getContext(), broadcast_type); odsState.addTypes(result); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { return create(builder, builder.getLoc(), std::forward(result), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = ::mlir::gpu::BroadcastTypeAttr::get(odsBuilder.getContext(), broadcast_type); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupBroadcastOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { return create(builder, builder.getLoc(), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { odsState.addOperands(src); if (lane) odsState.addOperands(lane); odsState.getOrAddProperties().broadcast_type = ::mlir::gpu::BroadcastTypeAttr::get(odsBuilder.getContext(), broadcast_type); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(src), std::forward(lane), std::forward(broadcast_type)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, /*optional*/::mlir::Value lane, ::mlir::gpu::BroadcastType broadcast_type) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(src), std::forward(lane), std::forward(broadcast_type)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupBroadcastOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupBroadcastOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupBroadcastOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupBroadcastOp SubgroupBroadcastOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupBroadcastOp::verifyInvariantsImpl() { auto tblgen_broadcast_type = getProperties().broadcast_type; (void)tblgen_broadcast_type; if (!tblgen_broadcast_type) return emitOpError("requires attribute 'broadcast_type'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps24(*this, tblgen_broadcast_type, "broadcast_type"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); if (valueGroup1.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup1.size(); } for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps17(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType()) && ((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType())))) return emitOpError("failed to verify that all of {result, src} have same type"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupBroadcastOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult SubgroupBroadcastOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupBroadcastOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc; (void)srcOperandsLoc; ::mlir::gpu::BroadcastTypeAttr broadcast_typeAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> laneOperands; ::llvm::SMLoc laneOperandsLoc; (void)laneOperandsLoc; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); srcOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(broadcast_typeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (broadcast_typeAttr) result.getOrAddProperties().broadcast_type = broadcast_typeAttr; { laneOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); laneOperands.push_back(operand); } } if (!laneOperands.empty()) { } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIntegerType(32); result.addTypes(resultTypes); if (parser.resolveOperands(srcOperands, resultTypes[0], srcOperandsLoc, result.operands)) return ::mlir::failure(); if (!laneOperands.empty()) { if (parser.resolveOperands(laneOperands, odsBuildableType0, laneOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SubgroupBroadcastOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrc(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getBroadcastTypeAttr()); if (getLane()) { _odsPrinter << ' '; if (::mlir::Value value = getLane()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("broadcast_type"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupBroadcastOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupBroadcastOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupBroadcastOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupIdOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupIdOpGenericAdaptorBase::SubgroupIdOpGenericAdaptorBase(SubgroupIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional< ::llvm::APInt > SubgroupIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail SubgroupIdOpAdaptor::SubgroupIdOpAdaptor(SubgroupIdOp op) : SubgroupIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.subgroup_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional SubgroupIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void SubgroupIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult SubgroupIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.upper_bound); } ::std::optional< ::llvm::APInt > SubgroupIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void SubgroupIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void SubgroupIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(result); } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(result), std::forward(upper_bound)); } void SubgroupIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(upper_bound)); } void SubgroupIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(upper_bound)); } void SubgroupIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupIdOp SubgroupIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupIdOp SubgroupIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupIdOp::verifyInvariantsImpl() { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SubgroupIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult SubgroupIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::IntegerAttr upper_boundAttr; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IndexType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); return ::mlir::success(); } void SubgroupIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IndexType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaComputeOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaComputeOpGenericAdaptorBase::SubgroupMmaComputeOpGenericAdaptorBase(SubgroupMmaComputeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional SubgroupMmaComputeOpGenericAdaptorBase::getATranspose() { auto attr = getATransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional SubgroupMmaComputeOpGenericAdaptorBase::getBTranspose() { auto attr = getBTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail SubgroupMmaComputeOpAdaptor::SubgroupMmaComputeOpAdaptor(SubgroupMmaComputeOp op) : SubgroupMmaComputeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaComputeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_a_transpose = getProperties().a_transpose; (void)tblgen_a_transpose; auto tblgen_b_transpose = getProperties().b_transpose; (void)tblgen_b_transpose; if (tblgen_a_transpose && !((::llvm::isa<::mlir::UnitAttr>(tblgen_a_transpose)))) return emitError(loc, "'gpu.subgroup_mma_compute' op attribute 'a_transpose' failed to satisfy constraint: unit attribute"); if (tblgen_b_transpose && !((::llvm::isa<::mlir::UnitAttr>(tblgen_b_transpose)))) return emitError(loc, "'gpu.subgroup_mma_compute' op attribute 'b_transpose' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaComputeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.a_transpose; auto attr = dict.get("a_transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `a_transpose` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.b_transpose; auto attr = dict.get("b_transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `b_transpose` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaComputeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.a_transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("a_transpose", propStorage)); } { const auto &propStorage = prop.b_transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("b_transpose", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaComputeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.a_transpose.getAsOpaquePointer()), llvm::hash_value(prop.b_transpose.getAsOpaquePointer())); } std::optional SubgroupMmaComputeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "a_transpose") return prop.a_transpose; if (name == "b_transpose") return prop.b_transpose; return std::nullopt; } void SubgroupMmaComputeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "a_transpose") { prop.a_transpose = ::llvm::dyn_cast_or_null>(value); return; } if (name == "b_transpose") { prop.b_transpose = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupMmaComputeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.a_transpose) attrs.append("a_transpose", prop.a_transpose); if (prop.b_transpose) attrs.append("b_transpose", prop.b_transpose); } ::llvm::LogicalResult SubgroupMmaComputeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getATransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "a_transpose", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getBTransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "b_transpose", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaComputeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.a_transpose))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.b_transpose))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaComputeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.a_transpose); writer.writeOptionalAttribute(prop.b_transpose); } ::std::optional SubgroupMmaComputeOp::getATranspose() { auto attr = getATransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional SubgroupMmaComputeOp::getBTranspose() { auto attr = getBTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void SubgroupMmaComputeOp::setATranspose(bool attrValue) { auto &odsProp = getProperties().a_transpose; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void SubgroupMmaComputeOp::setBTranspose(bool attrValue) { auto &odsProp = getProperties().b_transpose; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { odsState.addOperands(opA); odsState.addOperands(opB); odsState.addOperands(opC); if (a_transpose) { odsState.getOrAddProperties().a_transpose = a_transpose; } if (b_transpose) { odsState.getOrAddProperties().b_transpose = b_transpose; } odsState.addTypes(res); } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { return create(builder, builder.getLoc(), std::forward(res), std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { odsState.addOperands(opA); odsState.addOperands(opB); odsState.addOperands(opC); if (a_transpose) { odsState.getOrAddProperties().a_transpose = a_transpose; } if (b_transpose) { odsState.getOrAddProperties().b_transpose = b_transpose; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaComputeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { return create(builder, builder.getLoc(), std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { odsState.addOperands(opA); odsState.addOperands(opB); odsState.addOperands(opC); if (a_transpose) { odsState.getOrAddProperties().a_transpose = a_transpose; } if (b_transpose) { odsState.getOrAddProperties().b_transpose = b_transpose; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value opA, ::mlir::Value opB, ::mlir::Value opC, /*optional*/::mlir::UnitAttr a_transpose, /*optional*/::mlir::UnitAttr b_transpose) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(opA), std::forward(opB), std::forward(opC), std::forward(a_transpose), std::forward(b_transpose)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaComputeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupMmaComputeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaComputeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaComputeOp SubgroupMmaComputeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaComputeOp::verifyInvariantsImpl() { auto tblgen_a_transpose = getProperties().a_transpose; (void)tblgen_a_transpose; auto tblgen_b_transpose = getProperties().b_transpose; (void)tblgen_b_transpose; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_a_transpose, "a_transpose"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_b_transpose, "b_transpose"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps22(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps22(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps23(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(2).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that all of {opC, res} have same type"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaComputeOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult SubgroupMmaComputeOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 2) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[2].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupMmaComputeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand opARawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> opAOperands(&opARawOperand, 1); ::llvm::SMLoc opAOperandsLoc; (void)opAOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand opBRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> opBOperands(&opBRawOperand, 1); ::llvm::SMLoc opBOperandsLoc; (void)opBOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand opCRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> opCOperands(&opCRawOperand, 1); ::llvm::SMLoc opCOperandsLoc; (void)opCOperandsLoc; ::mlir::Type opARawType{}; ::llvm::ArrayRef<::mlir::Type> opATypes(&opARawType, 1); ::mlir::Type opBRawType{}; ::llvm::ArrayRef<::mlir::Type> opBTypes(&opBRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); opAOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(opARawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); opBOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(opBRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); opCOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(opCRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::gpu::MMAMatrixType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); opARawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::gpu::MMAMatrixType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); opBRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } result.addTypes(resTypes); if (parser.resolveOperands(opAOperands, opATypes, opAOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(opBOperands, opBTypes, opBOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(opCOperands, resTypes[0], opCOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaComputeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getOpA(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getOpB(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getOpC(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getOpA().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::gpu::MMAMatrixType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getOpB().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::gpu::MMAMatrixType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaComputeOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaComputeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupMmaComputeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaConstantMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaConstantMatrixOpGenericAdaptorBase::SubgroupMmaConstantMatrixOpGenericAdaptorBase(SubgroupMmaConstantMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail SubgroupMmaConstantMatrixOpAdaptor::SubgroupMmaConstantMatrixOpAdaptor(SubgroupMmaConstantMatrixOp op) : SubgroupMmaConstantMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaConstantMatrixOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaConstantMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaConstantMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaConstantMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SubgroupMmaConstantMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SubgroupMmaConstantMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SubgroupMmaConstantMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SubgroupMmaConstantMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SubgroupMmaConstantMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value value) { odsState.addOperands(value); odsState.addTypes(res); } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(res), std::forward(value)); } void SubgroupMmaConstantMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value) { odsState.addOperands(value); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value)); } void SubgroupMmaConstantMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaConstantMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaConstantMatrixOp SubgroupMmaConstantMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaConstantMatrixOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps25(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()).getElementType(), (*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that value type matches element type of mma_matrix"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaConstantMatrixOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SubgroupMmaConstantMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } for (::mlir::Type type : resTypes) { (void)type; if (!((::llvm::isa<::mlir::gpu::MMAMatrixType>(type)))) { return parser.emitError(parser.getNameLoc()) << "'res' must be MMAMatrix type, but got " << type; } } result.addTypes(resTypes); if (parser.resolveOperands(valueOperands, ::llvm::cast(resTypes[0]).getElementType(), valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaConstantMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaConstantMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaConstantMatrixOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaElementwiseOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaElementwiseOpGenericAdaptorBase::SubgroupMmaElementwiseOpGenericAdaptorBase(SubgroupMmaElementwiseOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupMmaElementwiseOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::gpu::MMAElementwiseOp SubgroupMmaElementwiseOpGenericAdaptorBase::getOpType() { auto attr = getOpTypeAttr(); return attr.getValue(); } } // namespace detail SubgroupMmaElementwiseOpAdaptor::SubgroupMmaElementwiseOpAdaptor(SubgroupMmaElementwiseOp op) : SubgroupMmaElementwiseOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaElementwiseOpAdaptor::verify(::mlir::Location loc) { auto tblgen_opType = getProperties().opType; (void)tblgen_opType; if (!tblgen_opType) return emitError(loc, "'gpu.subgroup_mma_elementwise' op requires attribute 'opType'"); if (tblgen_opType && !((::llvm::isa<::mlir::gpu::MMAElementwiseOpAttr>(tblgen_opType)))) return emitError(loc, "'gpu.subgroup_mma_elementwise' op attribute 'opType' failed to satisfy constraint: elementwise operation to apply to mma matrix"); return ::mlir::success(); } std::pair SubgroupMmaElementwiseOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupMmaElementwiseOp::getArgsMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupMmaElementwiseOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.opType; auto attr = dict.get("opType"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `opType` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaElementwiseOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.opType; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("opType", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaElementwiseOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.opType.getAsOpaquePointer())); } std::optional SubgroupMmaElementwiseOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "opType") return prop.opType; return std::nullopt; } void SubgroupMmaElementwiseOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "opType") { prop.opType = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupMmaElementwiseOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.opType) attrs.append("opType", prop.opType); } ::llvm::LogicalResult SubgroupMmaElementwiseOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getOpTypeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps25(attr, "opType", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaElementwiseOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.opType))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaElementwiseOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.opType); } ::mlir::gpu::MMAElementwiseOp SubgroupMmaElementwiseOp::getOpType() { auto attr = getOpTypeAttr(); return attr.getValue(); } void SubgroupMmaElementwiseOp::setOpType(::mlir::gpu::MMAElementwiseOp attrValue) { getProperties().opType = ::mlir::gpu::MMAElementwiseOpAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { odsState.addOperands(args); odsState.getOrAddProperties().opType = opType; odsState.addTypes(res); } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(args), std::forward(opType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { return create(builder, builder.getLoc(), std::forward(res), std::forward(args), std::forward(opType)); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { odsState.addOperands(args); odsState.getOrAddProperties().opType = opType; assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(args), std::forward(opType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOpAttr opType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(args), std::forward(opType)); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { odsState.addOperands(args); odsState.getOrAddProperties().opType = ::mlir::gpu::MMAElementwiseOpAttr::get(odsBuilder.getContext(), opType); odsState.addTypes(res); } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(args), std::forward(opType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { return create(builder, builder.getLoc(), std::forward(res), std::forward(args), std::forward(opType)); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { odsState.addOperands(args); odsState.getOrAddProperties().opType = ::mlir::gpu::MMAElementwiseOpAttr::get(odsBuilder.getContext(), opType); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(args), std::forward(opType)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange args, ::mlir::gpu::MMAElementwiseOp opType) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(args), std::forward(opType)); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaElementwiseOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaElementwiseOp SubgroupMmaElementwiseOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaElementwiseOp::verifyInvariantsImpl() { auto tblgen_opType = getProperties().opType; (void)tblgen_opType; if (!tblgen_opType) return emitOpError("requires attribute 'opType'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps25(*this, tblgen_opType, "opType"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps26(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((true))) return emitOpError("failed to verify that all of {args} have same type"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaElementwiseOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SubgroupMmaElementwiseOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::MMAElementwiseOpAttr opTypeAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> argsOperands; ::llvm::SMLoc argsOperandsLoc; (void)argsOperandsLoc; ::llvm::ArrayRef<::mlir::Type> argsTypes; ::llvm::ArrayRef<::mlir::Type> resTypes; if (parser.parseCustomAttributeWithFallback(opTypeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (opTypeAttr) result.getOrAddProperties().opType = opTypeAttr; argsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(argsOperands)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); ::mlir::FunctionType args__res_functionType; if (parser.parseType(args__res_functionType)) return ::mlir::failure(); argsTypes = args__res_functionType.getInputs(); resTypes = args__res_functionType.getResults(); result.addTypes(resTypes); if (parser.resolveOperands(argsOperands, argsTypes, argsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaElementwiseOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getOpTypeAttr()); _odsPrinter << ' '; _odsPrinter << getArgs(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("opType"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printFunctionalType(getArgs().getTypes(), ::llvm::ArrayRef<::mlir::Type>(getRes().getType())); } void SubgroupMmaElementwiseOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaElementwiseOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupMmaElementwiseOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaExtractThreadLocalOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaExtractThreadLocalOpGenericAdaptorBase::SubgroupMmaExtractThreadLocalOpGenericAdaptorBase(SubgroupMmaExtractThreadLocalOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupMmaExtractThreadLocalOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SubgroupMmaExtractThreadLocalOpAdaptor::SubgroupMmaExtractThreadLocalOpAdaptor(SubgroupMmaExtractThreadLocalOp op) : SubgroupMmaExtractThreadLocalOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SubgroupMmaExtractThreadLocalOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupMmaExtractThreadLocalOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaExtractThreadLocalOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaExtractThreadLocalOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SubgroupMmaExtractThreadLocalOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SubgroupMmaExtractThreadLocalOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SubgroupMmaExtractThreadLocalOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value matrix, ::mlir::ValueRange indices) { odsState.addOperands(matrix); odsState.addOperands(indices); odsState.addTypes(res); } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value matrix, ::mlir::ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(matrix), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value matrix, ::mlir::ValueRange indices) { return create(builder, builder.getLoc(), std::forward(res), std::forward(matrix), std::forward(indices)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value matrix, ::mlir::ValueRange indices) { odsState.addOperands(matrix); odsState.addOperands(indices); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaExtractThreadLocalOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value matrix, ::mlir::ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(matrix), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value matrix, ::mlir::ValueRange indices) { return create(builder, builder.getLoc(), std::forward(matrix), std::forward(indices)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value matrix, ::mlir::ValueRange indices) { odsState.addOperands(matrix); odsState.addOperands(indices); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value matrix, ::mlir::ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(matrix), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value matrix, ::mlir::ValueRange indices) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(matrix), std::forward(indices)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaExtractThreadLocalOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupMmaExtractThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupMmaExtractThreadLocalOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaExtractThreadLocalOp SubgroupMmaExtractThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).getElementType(), (*this->getODSResults(0).begin()).getType())))) return emitOpError("failed to verify that value type matches element type of mma_matrix"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult SubgroupMmaExtractThreadLocalOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = ::llvm::cast(operands[0].getType()).getElementType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupMmaExtractThreadLocalOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand matrixRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixOperands(&matrixRawOperand, 1); ::llvm::SMLoc matrixOperandsLoc; (void)matrixOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::Type matrixRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixTypes(&matrixRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); matrixOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resTypes); if (parser.resolveOperands(matrixOperands, matrixTypes, matrixOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaExtractThreadLocalOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getMatrix(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getMatrix().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaExtractThreadLocalOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaExtractThreadLocalOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaInsertThreadLocalOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaInsertThreadLocalOpGenericAdaptorBase::SubgroupMmaInsertThreadLocalOpGenericAdaptorBase(SubgroupMmaInsertThreadLocalOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupMmaInsertThreadLocalOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail SubgroupMmaInsertThreadLocalOpAdaptor::SubgroupMmaInsertThreadLocalOpAdaptor(SubgroupMmaInsertThreadLocalOp op) : SubgroupMmaInsertThreadLocalOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaInsertThreadLocalOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SubgroupMmaInsertThreadLocalOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupMmaInsertThreadLocalOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupMmaInsertThreadLocalOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaInsertThreadLocalOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaInsertThreadLocalOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional SubgroupMmaInsertThreadLocalOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void SubgroupMmaInsertThreadLocalOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void SubgroupMmaInsertThreadLocalOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult SubgroupMmaInsertThreadLocalOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void SubgroupMmaInsertThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { odsState.addOperands(value); odsState.addOperands(matrix); odsState.addOperands(indices); odsState.addTypes(res); } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(value), std::forward(matrix), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { return create(builder, builder.getLoc(), std::forward(res), std::forward(value), std::forward(matrix), std::forward(indices)); } void SubgroupMmaInsertThreadLocalOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { odsState.addOperands(value); odsState.addOperands(matrix); odsState.addOperands(indices); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(matrix), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value matrix, ::mlir::ValueRange indices) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(matrix), std::forward(indices)); } void SubgroupMmaInsertThreadLocalOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaInsertThreadLocalOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaInsertThreadLocalOp SubgroupMmaInsertThreadLocalOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaInsertThreadLocalOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).getElementType(), (*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that value type matches element type of mma_matrix"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaInsertThreadLocalOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SubgroupMmaInsertThreadLocalOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand matrixRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> matrixOperands(&matrixRawOperand, 1); ::llvm::SMLoc matrixOperandsLoc; (void)matrixOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); ::mlir::Type matrixRawType{}; ::llvm::ArrayRef<::mlir::Type> matrixTypes(&matrixRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); matrixOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(matrixRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); matrixRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resTypes); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(matrixOperands, matrixTypes, matrixOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaInsertThreadLocalOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMatrix(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMatrix().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaInsertThreadLocalOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaInsertThreadLocalOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaLoadMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaLoadMatrixOpGenericAdaptorBase::SubgroupMmaLoadMatrixOpGenericAdaptorBase(SubgroupMmaLoadMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupMmaLoadMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::APInt SubgroupMmaLoadMatrixOpGenericAdaptorBase::getLeadDimension() { auto attr = getLeadDimensionAttr(); return attr.getValue(); } ::std::optional SubgroupMmaLoadMatrixOpGenericAdaptorBase::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail SubgroupMmaLoadMatrixOpAdaptor::SubgroupMmaLoadMatrixOpAdaptor(SubgroupMmaLoadMatrixOp op) : SubgroupMmaLoadMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaLoadMatrixOpAdaptor::verify(::mlir::Location loc) { auto tblgen_leadDimension = getProperties().leadDimension; (void)tblgen_leadDimension; if (!tblgen_leadDimension) return emitError(loc, "'gpu.subgroup_mma_load_matrix' op requires attribute 'leadDimension'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (tblgen_leadDimension && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_leadDimension))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_leadDimension).getType()))))) return emitError(loc, "'gpu.subgroup_mma_load_matrix' op attribute 'leadDimension' failed to satisfy constraint: index attribute"); if (tblgen_transpose && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transpose)))) return emitError(loc, "'gpu.subgroup_mma_load_matrix' op attribute 'transpose' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } std::pair SubgroupMmaLoadMatrixOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupMmaLoadMatrixOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupMmaLoadMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.leadDimension; auto attr = dict.get("leadDimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `leadDimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transpose; auto attr = dict.get("transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transpose` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaLoadMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.leadDimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("leadDimension", propStorage)); } { const auto &propStorage = prop.transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transpose", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaLoadMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.leadDimension.getAsOpaquePointer()), llvm::hash_value(prop.transpose.getAsOpaquePointer())); } std::optional SubgroupMmaLoadMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "leadDimension") return prop.leadDimension; if (name == "transpose") return prop.transpose; return std::nullopt; } void SubgroupMmaLoadMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "leadDimension") { prop.leadDimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "transpose") { prop.transpose = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupMmaLoadMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.leadDimension) attrs.append("leadDimension", prop.leadDimension); if (prop.transpose) attrs.append("transpose", prop.transpose); } ::llvm::LogicalResult SubgroupMmaLoadMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLeadDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "leadDimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "transpose", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaLoadMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.leadDimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.transpose))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaLoadMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.leadDimension); writer.writeOptionalAttribute(prop.transpose); } ::llvm::APInt SubgroupMmaLoadMatrixOp::getLeadDimension() { auto attr = getLeadDimensionAttr(); return attr.getValue(); } ::std::optional SubgroupMmaLoadMatrixOp::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void SubgroupMmaLoadMatrixOp::setLeadDimension(::llvm::APInt attrValue) { getProperties().leadDimension = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), attrValue); } void SubgroupMmaLoadMatrixOp::setTranspose(bool attrValue) { auto &odsProp = getProperties().transpose; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = leadDimension; if (transpose) { odsState.getOrAddProperties().transpose = transpose; } odsState.addTypes(res); } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = leadDimension; if (transpose) { odsState.getOrAddProperties().transpose = transpose; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), leadDimension); if (transpose) { odsState.getOrAddProperties().transpose = transpose; } odsState.addTypes(res); } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(res), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(srcMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), leadDimension); if (transpose) { odsState.getOrAddProperties().transpose = transpose; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value srcMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(srcMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaLoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaLoadMatrixOp SubgroupMmaLoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaLoadMatrixOp::verifyInvariantsImpl() { auto tblgen_leadDimension = getProperties().leadDimension; (void)tblgen_leadDimension; if (!tblgen_leadDimension) return emitOpError("requires attribute 'leadDimension'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_leadDimension, "leadDimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_transpose, "transpose"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps27(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps24(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaLoadMatrixOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult SubgroupMmaLoadMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcMemrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcMemrefOperands(&srcMemrefRawOperand, 1); ::llvm::SMLoc srcMemrefOperandsLoc; (void)srcMemrefOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::Type srcMemrefRawType{}; ::llvm::ArrayRef<::mlir::Type> srcMemrefTypes(&srcMemrefRawType, 1); ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); srcMemrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcMemrefRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcMemrefRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resTypes); if (parser.resolveOperands(srcMemrefOperands, srcMemrefTypes, srcMemrefOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaLoadMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrcMemref(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSrcMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaLoadMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaLoadMatrixOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupMmaLoadMatrixOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupMmaStoreMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupMmaStoreMatrixOpGenericAdaptorBase::SubgroupMmaStoreMatrixOpGenericAdaptorBase(SubgroupMmaStoreMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SubgroupMmaStoreMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::APInt SubgroupMmaStoreMatrixOpGenericAdaptorBase::getLeadDimension() { auto attr = getLeadDimensionAttr(); return attr.getValue(); } ::std::optional SubgroupMmaStoreMatrixOpGenericAdaptorBase::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } } // namespace detail SubgroupMmaStoreMatrixOpAdaptor::SubgroupMmaStoreMatrixOpAdaptor(SubgroupMmaStoreMatrixOp op) : SubgroupMmaStoreMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupMmaStoreMatrixOpAdaptor::verify(::mlir::Location loc) { auto tblgen_leadDimension = getProperties().leadDimension; (void)tblgen_leadDimension; if (!tblgen_leadDimension) return emitError(loc, "'gpu.subgroup_mma_store_matrix' op requires attribute 'leadDimension'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (tblgen_leadDimension && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_leadDimension))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_leadDimension).getType()))))) return emitError(loc, "'gpu.subgroup_mma_store_matrix' op attribute 'leadDimension' failed to satisfy constraint: index attribute"); if (tblgen_transpose && !((::llvm::isa<::mlir::UnitAttr>(tblgen_transpose)))) return emitError(loc, "'gpu.subgroup_mma_store_matrix' op attribute 'transpose' failed to satisfy constraint: unit attribute"); return ::mlir::success(); } std::pair SubgroupMmaStoreMatrixOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange SubgroupMmaStoreMatrixOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult SubgroupMmaStoreMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.leadDimension; auto attr = dict.get("leadDimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `leadDimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transpose; auto attr = dict.get("transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transpose` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupMmaStoreMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.leadDimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("leadDimension", propStorage)); } { const auto &propStorage = prop.transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transpose", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupMmaStoreMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.leadDimension.getAsOpaquePointer()), llvm::hash_value(prop.transpose.getAsOpaquePointer())); } std::optional SubgroupMmaStoreMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "leadDimension") return prop.leadDimension; if (name == "transpose") return prop.transpose; return std::nullopt; } void SubgroupMmaStoreMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "leadDimension") { prop.leadDimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "transpose") { prop.transpose = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupMmaStoreMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.leadDimension) attrs.append("leadDimension", prop.leadDimension); if (prop.transpose) attrs.append("transpose", prop.transpose); } ::llvm::LogicalResult SubgroupMmaStoreMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLeadDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "leadDimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "transpose", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaStoreMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.leadDimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.transpose))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaStoreMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.leadDimension); writer.writeOptionalAttribute(prop.transpose); } ::llvm::APInt SubgroupMmaStoreMatrixOp::getLeadDimension() { auto attr = getLeadDimensionAttr(); return attr.getValue(); } ::std::optional SubgroupMmaStoreMatrixOp::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } void SubgroupMmaStoreMatrixOp::setLeadDimension(::llvm::APInt attrValue) { getProperties().leadDimension = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), attrValue); } void SubgroupMmaStoreMatrixOp::setTranspose(bool attrValue) { auto &odsProp = getProperties().transpose; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(src); odsState.addOperands(dstMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = leadDimension; if (transpose) { odsState.getOrAddProperties().transpose = transpose; } } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(src); odsState.addOperands(dstMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = leadDimension; if (transpose) { odsState.getOrAddProperties().transpose = transpose; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::mlir::IntegerAttr leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(src); odsState.addOperands(dstMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), leadDimension); if (transpose) { odsState.getOrAddProperties().transpose = transpose; } } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { odsState.addOperands(src); odsState.addOperands(dstMemref); odsState.addOperands(indices); odsState.getOrAddProperties().leadDimension = odsBuilder.getIntegerAttr(odsBuilder.getIndexType(), leadDimension); if (transpose) { odsState.getOrAddProperties().transpose = transpose; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value src, ::mlir::Value dstMemref, ::mlir::ValueRange indices, ::llvm::APInt leadDimension, /*optional*/::mlir::UnitAttr transpose) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(src), std::forward(dstMemref), std::forward(indices), std::forward(leadDimension), std::forward(transpose)); } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupMmaStoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupMmaStoreMatrixOp SubgroupMmaStoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupMmaStoreMatrixOp::verifyInvariantsImpl() { auto tblgen_leadDimension = getProperties().leadDimension; (void)tblgen_leadDimension; if (!tblgen_leadDimension) return emitOpError("requires attribute 'leadDimension'"); auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_leadDimension, "leadDimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_transpose, "transpose"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps28(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps27(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SubgroupMmaStoreMatrixOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult SubgroupMmaStoreMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand srcRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> srcOperands(&srcRawOperand, 1); ::llvm::SMLoc srcOperandsLoc; (void)srcOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand dstMemrefRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dstMemrefOperands(&dstMemrefRawOperand, 1); ::llvm::SMLoc dstMemrefOperandsLoc; (void)dstMemrefOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::Type srcRawType{}; ::llvm::ArrayRef<::mlir::Type> srcTypes(&srcRawType, 1); ::mlir::Type dstMemrefRawType{}; ::llvm::ArrayRef<::mlir::Type> dstMemrefTypes(&dstMemrefRawType, 1); srcOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(srcRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); dstMemrefOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dstMemrefRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::gpu::MMAMatrixType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); srcRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); dstMemrefRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(srcOperands, srcTypes, srcOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(dstMemrefOperands, dstMemrefTypes, dstMemrefOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupMmaStoreMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSrc(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDstMemref(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSrc().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::gpu::MMAMatrixType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getDstMemref().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupMmaStoreMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, true, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupMmaStoreMatrixOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupMmaStoreMatrixOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupReduceOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupReduceOpGenericAdaptorBase::SubgroupReduceOpGenericAdaptorBase(SubgroupReduceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::AllReduceOperation SubgroupReduceOpGenericAdaptorBase::getOp() { auto attr = getOpAttr(); return attr.getValue(); } ::mlir::UnitAttr SubgroupReduceOpGenericAdaptorBase::getUniformAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::UnitAttr>(getProperties().uniform); if (!attr) attr = ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr); return attr; } bool SubgroupReduceOpGenericAdaptorBase::getUniform() { auto attr = getUniformAttr(); if (!attr) return ((false) ? ::mlir::Builder(odsAttrs.getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } ::std::optional SubgroupReduceOpGenericAdaptorBase::getClusterSize() { auto attr = getClusterSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::mlir::IntegerAttr SubgroupReduceOpGenericAdaptorBase::getClusterStrideAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::IntegerAttr>(getProperties().cluster_stride); return attr; } uint32_t SubgroupReduceOpGenericAdaptorBase::getClusterStride() { auto attr = getClusterStrideAttr(); return attr.getValue().getZExtValue(); } } // namespace detail SubgroupReduceOpAdaptor::SubgroupReduceOpAdaptor(SubgroupReduceOp op) : SubgroupReduceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupReduceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_cluster_size = getProperties().cluster_size; (void)tblgen_cluster_size; auto tblgen_cluster_stride = getProperties().cluster_stride; (void)tblgen_cluster_stride; auto tblgen_op = getProperties().op; (void)tblgen_op; if (!tblgen_op) return emitError(loc, "'gpu.subgroup_reduce' op requires attribute 'op'"); auto tblgen_uniform = getProperties().uniform; (void)tblgen_uniform; if (tblgen_op && !((::llvm::isa<::mlir::gpu::AllReduceOperationAttr>(tblgen_op)))) return emitError(loc, "'gpu.subgroup_reduce' op attribute 'op' failed to satisfy constraint: built-in reduction operations supported by gpu.allreduce."); if (tblgen_uniform && !((::llvm::isa<::mlir::UnitAttr>(tblgen_uniform)))) return emitError(loc, "'gpu.subgroup_reduce' op attribute 'uniform' failed to satisfy constraint: unit attribute"); if (tblgen_cluster_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_cluster_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_cluster_size).getType().isSignlessInteger(32))))) return emitError(loc, "'gpu.subgroup_reduce' op attribute 'cluster_size' failed to satisfy constraint: 32-bit signless integer attribute"); if (tblgen_cluster_stride && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_cluster_stride))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_cluster_stride).getType().isSignlessInteger(32))))) return emitError(loc, "'gpu.subgroup_reduce' op attribute 'cluster_stride' failed to satisfy constraint: 32-bit signless integer attribute"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupReduceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.cluster_size; auto attr = dict.get("cluster_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `cluster_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.cluster_stride; auto attr = dict.get("cluster_stride"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `cluster_stride` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.op; auto attr = dict.get("op"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `op` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.uniform; auto attr = dict.get("uniform"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `uniform` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupReduceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.cluster_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("cluster_size", propStorage)); } { const auto &propStorage = prop.cluster_stride; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("cluster_stride", propStorage)); } { const auto &propStorage = prop.op; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("op", propStorage)); } { const auto &propStorage = prop.uniform; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("uniform", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupReduceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.cluster_size.getAsOpaquePointer()), llvm::hash_value(prop.cluster_stride.getAsOpaquePointer()), llvm::hash_value(prop.op.getAsOpaquePointer()), llvm::hash_value(prop.uniform.getAsOpaquePointer())); } std::optional SubgroupReduceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "cluster_size") return prop.cluster_size; if (name == "cluster_stride") return prop.cluster_stride; if (name == "op") return prop.op; if (name == "uniform") return prop.uniform; return std::nullopt; } void SubgroupReduceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "cluster_size") { prop.cluster_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "cluster_stride") { prop.cluster_stride = ::llvm::dyn_cast_or_null>(value); return; } if (name == "op") { prop.op = ::llvm::dyn_cast_or_null>(value); return; } if (name == "uniform") { prop.uniform = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupReduceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.cluster_size) attrs.append("cluster_size", prop.cluster_size); if (prop.cluster_stride) attrs.append("cluster_stride", prop.cluster_stride); if (prop.op) attrs.append("op", prop.op); if (prop.uniform) attrs.append("uniform", prop.uniform); } ::llvm::LogicalResult SubgroupReduceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getClusterSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps26(attr, "cluster_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getClusterStrideAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps26(attr, "cluster_stride", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getOpAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps1(attr, "op", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUniformAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(attr, "uniform", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupReduceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.cluster_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.cluster_stride))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.op))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.uniform))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupReduceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.cluster_size); writer.writeOptionalAttribute(prop.cluster_stride); writer.writeAttribute(prop.op); writer.writeOptionalAttribute(prop.uniform); } ::mlir::gpu::AllReduceOperation SubgroupReduceOp::getOp() { auto attr = getOpAttr(); return attr.getValue(); } bool SubgroupReduceOp::getUniform() { auto attr = getUniformAttr(); if (!attr) return ((false) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr) != nullptr; return attr != nullptr; } ::std::optional SubgroupReduceOp::getClusterSize() { auto attr = getClusterSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } uint32_t SubgroupReduceOp::getClusterStride() { auto attr = getClusterStrideAttr(); return attr.getValue().getZExtValue(); } void SubgroupReduceOp::setOp(::mlir::gpu::AllReduceOperation attrValue) { getProperties().op = ::mlir::gpu::AllReduceOperationAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void SubgroupReduceOp::setUniform(bool attrValue) { auto &odsProp = getProperties().uniform; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void SubgroupReduceOp::setClusterSize(::std::optional attrValue) { auto &odsProp = getProperties().cluster_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), *attrValue); else odsProp = nullptr; } void SubgroupReduceOp::setClusterStride(uint32_t attrValue) { getProperties().cluster_stride = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(32), attrValue); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform) { build(odsBuilder, odsState, value, op, uniform, std::nullopt); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size) { build(odsBuilder, odsState, value, op, uniform, cluster_size ? odsBuilder.getI32IntegerAttr(*cluster_size) : nullptr); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size, uint32_t cluster_stride) { build(odsBuilder, odsState, value, op, uniform, cluster_size ? odsBuilder.getI32IntegerAttr(*cluster_size) : nullptr, cluster_stride); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size, uint32_t cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, ::mlir::gpu::AllReduceOperation op, bool uniform, std::optional cluster_size, uint32_t cluster_stride) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = op; if (uniform) { odsState.getOrAddProperties().uniform = uniform; } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } if (cluster_stride) { odsState.getOrAddProperties().cluster_stride = cluster_stride; } odsState.addTypes(result); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { return create(builder, builder.getLoc(), std::forward(result), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = op; if (uniform) { odsState.getOrAddProperties().uniform = uniform; } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } if (cluster_stride) { odsState.getOrAddProperties().cluster_stride = cluster_stride; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = op; if (uniform) { odsState.getOrAddProperties().uniform = uniform; } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } if (cluster_stride) { odsState.getOrAddProperties().cluster_stride = cluster_stride; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperationAttr op, /*optional*/::mlir::UnitAttr uniform, /*optional*/::mlir::IntegerAttr cluster_size, ::mlir::IntegerAttr cluster_stride) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = ::mlir::gpu::AllReduceOperationAttr::get(odsBuilder.getContext(), op); if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } odsState.getOrAddProperties().cluster_stride = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), cluster_stride); odsState.addTypes(result); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { return create(builder, builder.getLoc(), std::forward(result), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = ::mlir::gpu::AllReduceOperationAttr::get(odsBuilder.getContext(), op); if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } odsState.getOrAddProperties().cluster_stride = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), cluster_stride); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { return create(builder, builder.getLoc(), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { odsState.addOperands(value); odsState.getOrAddProperties().op = ::mlir::gpu::AllReduceOperationAttr::get(odsBuilder.getContext(), op); if (uniform) { odsState.getOrAddProperties().uniform = ((uniform) ? odsBuilder.getUnitAttr() : nullptr); } if (cluster_size) { odsState.getOrAddProperties().cluster_size = cluster_size; } odsState.getOrAddProperties().cluster_stride = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), cluster_stride); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::gpu::AllReduceOperation op, /*optional*/bool uniform, /*optional*/::mlir::IntegerAttr cluster_size, uint32_t cluster_stride) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(op), std::forward(uniform), std::forward(cluster_size), std::forward(cluster_stride)); } void SubgroupReduceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupReduceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupReduceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupReduceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupReduceOp SubgroupReduceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupReduceOp SubgroupReduceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupReduceOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.cluster_stride) properties.cluster_stride = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 1); } ::llvm::LogicalResult SubgroupReduceOp::verifyInvariantsImpl() { auto tblgen_cluster_size = getProperties().cluster_size; (void)tblgen_cluster_size; auto tblgen_cluster_stride = getProperties().cluster_stride; (void)tblgen_cluster_stride; auto tblgen_op = getProperties().op; (void)tblgen_op; if (!tblgen_op) return emitOpError("requires attribute 'op'"); auto tblgen_uniform = getProperties().uniform; (void)tblgen_uniform; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps1(*this, tblgen_op, "op"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps2(*this, tblgen_uniform, "uniform"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps26(*this, tblgen_cluster_size, "cluster_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps26(*this, tblgen_cluster_stride, "cluster_stride"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps19(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SubgroupReduceOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult SubgroupReduceOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupReduceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::AllReduceOperationAttr opAttr; ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::IntegerAttr cluster_sizeAttr; ::mlir::IntegerAttr cluster_strideAttr; ::llvm::ArrayRef<::mlir::Type> allOperandTypes; ::llvm::ArrayRef<::mlir::Type> allResultTypes; { auto odsResult = parseAllReduceOperation(parser, opAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().op = opAttr; } valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("uniform"))) { result.getOrAddProperties().uniform = parser.getBuilder().getUnitAttr(); } if (::mlir::succeeded(parser.parseOptionalKeyword("cluster"))) { if (parser.parseLParen()) return ::mlir::failure(); if (parser.parseKeyword("size")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(cluster_sizeAttr, parser.getBuilder().getIntegerType(32))) { return ::mlir::failure(); } if (cluster_sizeAttr) result.getOrAddProperties().cluster_size = cluster_sizeAttr; if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("stride")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(cluster_strideAttr, parser.getBuilder().getIntegerType(32))) { return ::mlir::failure(); } if (cluster_strideAttr) result.getOrAddProperties().cluster_stride = cluster_strideAttr; } if (parser.parseRParen()) return ::mlir::failure(); } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); ::mlir::FunctionType allOperand__allResult_functionType; if (parser.parseType(allOperand__allResult_functionType)) return ::mlir::failure(); allOperandTypes = allOperand__allResult_functionType.getInputs(); allResultTypes = allOperand__allResult_functionType.getResults(); result.addTypes(allResultTypes); if (parser.resolveOperands(valueOperands, allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } void SubgroupReduceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAllReduceOperation(_odsPrinter, *this, getOpAttr()); _odsPrinter << ' '; _odsPrinter << getValue(); if ((getUniformAttr() && getUniformAttr() != ((false) ? ::mlir::OpBuilder((*this)->getContext()).getUnitAttr() : nullptr))) { _odsPrinter << ' ' << "uniform"; } if (getClusterSizeAttr()) { _odsPrinter << ' ' << "cluster"; _odsPrinter << "("; _odsPrinter << "size"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getClusterSizeAttr()); if (getClusterStrideAttr() != ::mlir::OpBuilder((*this)->getContext()).getIntegerAttr(::mlir::OpBuilder((*this)->getContext()).getIntegerType(32), 1)) { _odsPrinter << ","; _odsPrinter << ' ' << "stride"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getClusterStrideAttr()); } _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("op"); elidedAttrs.push_back("uniform"); elidedAttrs.push_back("cluster_size"); elidedAttrs.push_back("cluster_stride"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getUniformAttr(); if(attr && (attr == ((false) ? odsBuilder.getUnitAttr() : nullptr))) elidedAttrs.push_back("uniform"); } { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getClusterStrideAttr(); if(attr && (attr == odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(32), 1))) elidedAttrs.push_back("cluster_stride"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printFunctionalType(getOperation()->getOperandTypes(), getOperation()->getResultTypes()); } void SubgroupReduceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupReduceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupReduceOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::SubgroupSizeOp definitions //===----------------------------------------------------------------------===// namespace detail { SubgroupSizeOpGenericAdaptorBase::SubgroupSizeOpGenericAdaptorBase(SubgroupSizeOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional< ::llvm::APInt > SubgroupSizeOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail SubgroupSizeOpAdaptor::SubgroupSizeOpAdaptor(SubgroupSizeOp op) : SubgroupSizeOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SubgroupSizeOpAdaptor::verify(::mlir::Location loc) { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.subgroup_size' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult SubgroupSizeOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute SubgroupSizeOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SubgroupSizeOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional SubgroupSizeOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void SubgroupSizeOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void SubgroupSizeOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult SubgroupSizeOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult SubgroupSizeOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void SubgroupSizeOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.upper_bound); } ::std::optional< ::llvm::APInt > SubgroupSizeOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void SubgroupSizeOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void SubgroupSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(result); } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(result), std::forward(upper_bound)); } void SubgroupSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupSizeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(upper_bound)); } void SubgroupSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(upper_bound)); } void SubgroupSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SubgroupSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupSizeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void SubgroupSizeOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void SubgroupSizeOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(SubgroupSizeOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } SubgroupSizeOp SubgroupSizeOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SubgroupSizeOp SubgroupSizeOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SubgroupSizeOp::verifyInvariantsImpl() { auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult SubgroupSizeOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult SubgroupSizeOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult SubgroupSizeOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::IntegerAttr upper_boundAttr; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IndexType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); return ::mlir::success(); } void SubgroupSizeOp::print(::mlir::OpAsmPrinter &_odsPrinter) { if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IndexType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SubgroupSizeOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::SubgroupSizeOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::SubgroupSizeOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::TerminatorOp definitions //===----------------------------------------------------------------------===// namespace detail { TerminatorOpGenericAdaptorBase::TerminatorOpGenericAdaptorBase(TerminatorOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail TerminatorOpAdaptor::TerminatorOpAdaptor(TerminatorOp op) : TerminatorOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TerminatorOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult TerminatorOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TerminatorOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TerminatorOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional TerminatorOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void TerminatorOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void TerminatorOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult TerminatorOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void TerminatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState) { } TerminatorOp TerminatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TerminatorOp TerminatorOp::create(::mlir::ImplicitLocOpBuilder &builder) { return create(builder, builder.getLoc()); } void TerminatorOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TerminatorOp TerminatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TerminatorOp TerminatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void TerminatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TerminatorOp TerminatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TerminatorOp TerminatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TerminatorOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TerminatorOp TerminatorOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TerminatorOp TerminatorOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TerminatorOp::verifyInvariantsImpl() { return ::mlir::success(); } ::llvm::LogicalResult TerminatorOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult TerminatorOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } return ::mlir::success(); } void TerminatorOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void TerminatorOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::TerminatorOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::ThreadIdOp definitions //===----------------------------------------------------------------------===// namespace detail { ThreadIdOpGenericAdaptorBase::ThreadIdOpGenericAdaptorBase(ThreadIdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::gpu::Dimension ThreadIdOpGenericAdaptorBase::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ThreadIdOpGenericAdaptorBase::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } } // namespace detail ThreadIdOpAdaptor::ThreadIdOpAdaptor(ThreadIdOp op) : ThreadIdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ThreadIdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitError(loc, "'gpu.thread_id' op requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (tblgen_dimension && !((::llvm::isa<::mlir::gpu::DimensionAttr>(tblgen_dimension)))) return emitError(loc, "'gpu.thread_id' op attribute 'dimension' failed to satisfy constraint: a dimension, either 'x', 'y', or 'z'"); if (tblgen_upper_bound && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_upper_bound))) && ((::llvm::isa<::mlir::IndexType>(::llvm::cast<::mlir::IntegerAttr>(tblgen_upper_bound).getType()))))) return emitError(loc, "'gpu.thread_id' op attribute 'upper_bound' failed to satisfy constraint: index attribute"); return ::mlir::success(); } ::llvm::LogicalResult ThreadIdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.dimension; auto attr = dict.get("dimension"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `dimension` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.upper_bound; auto attr = dict.get("upper_bound"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `upper_bound` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ThreadIdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.dimension; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("dimension", propStorage)); } { const auto &propStorage = prop.upper_bound; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("upper_bound", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ThreadIdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.dimension.getAsOpaquePointer()), llvm::hash_value(prop.upper_bound.getAsOpaquePointer())); } std::optional ThreadIdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "dimension") return prop.dimension; if (name == "upper_bound") return prop.upper_bound; return std::nullopt; } void ThreadIdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "dimension") { prop.dimension = ::llvm::dyn_cast_or_null>(value); return; } if (name == "upper_bound") { prop.upper_bound = ::llvm::dyn_cast_or_null>(value); return; } } void ThreadIdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.dimension) attrs.append("dimension", prop.dimension); if (prop.upper_bound) attrs.append("upper_bound", prop.upper_bound); } ::llvm::LogicalResult ThreadIdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getDimensionAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(attr, "dimension", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getUpperBoundAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(attr, "upper_bound", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ThreadIdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.dimension))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.upper_bound))) return ::mlir::failure(); return ::mlir::success(); } void ThreadIdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.dimension); writer.writeOptionalAttribute(prop.upper_bound); } ::mlir::gpu::Dimension ThreadIdOp::getDimension() { auto attr = getDimensionAttr(); return attr.getValue(); } ::std::optional< ::llvm::APInt > ThreadIdOp::getUpperBound() { auto attr = getUpperBoundAttr(); return attr ? ::std::optional< ::llvm::APInt >(attr.getValue()) : (::std::nullopt); } void ThreadIdOp::setDimension(::mlir::gpu::Dimension attrValue) { getProperties().dimension = ::mlir::gpu::DimensionAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ThreadIdOp::setUpperBound(::std::optional<::llvm::APInt> attrValue) { auto &odsProp = getProperties().upper_bound; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIndexType(), *attrValue); else odsProp = nullptr; } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, dimension, /*upperBound=*/nullptr); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(dimension)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { build(odsBuilder, odsState, resultType, dimension, /*upperBound=*/nullptr); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(dimension)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType, ::mlir::gpu::Dimension dimension) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(dimension)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ThreadIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = dimension; if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::DimensionAttr dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } odsState.addTypes(resultType0); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ThreadIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { odsState.getOrAddProperties().dimension = ::mlir::gpu::DimensionAttr::get(odsBuilder.getContext(), dimension); if (upper_bound) { odsState.getOrAddProperties().upper_bound = upper_bound; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::gpu::Dimension dimension, /*optional*/::mlir::IntegerAttr upper_bound) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(dimension), std::forward(upper_bound)); } void ThreadIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ThreadIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ThreadIdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ThreadIdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ThreadIdOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ThreadIdOp ThreadIdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ThreadIdOp ThreadIdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ThreadIdOp::verifyInvariantsImpl() { auto tblgen_dimension = getProperties().dimension; (void)tblgen_dimension; if (!tblgen_dimension) return emitOpError("requires attribute 'dimension'"); auto tblgen_upper_bound = getProperties().upper_bound; (void)tblgen_upper_bound; if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps8(*this, tblgen_dimension, "dimension"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps9(*this, tblgen_upper_bound, "upper_bound"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ThreadIdOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ThreadIdOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ThreadIdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::gpu::DimensionAttr dimensionAttr; ::mlir::IntegerAttr upper_boundAttr; if (parser.parseCustomAttributeWithFallback(dimensionAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (dimensionAttr) result.getOrAddProperties().dimension = dimensionAttr; if (::mlir::succeeded(parser.parseOptionalKeyword("upper_bound"))) { if (parser.parseCustomAttributeWithFallback(upper_boundAttr, parser.getBuilder().getIndexType())) { return ::mlir::failure(); } if (upper_boundAttr) result.getOrAddProperties().upper_bound = upper_boundAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void ThreadIdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getDimensionAttr()); if (getUpperBoundAttr()) { _odsPrinter << ' ' << "upper_bound"; _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getUpperBoundAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("dimension"); elidedAttrs.push_back("upper_bound"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void ThreadIdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } void ThreadIdOp::getAsmResultNames( llvm::function_ref setNameFn) { auto dimStr = stringifyDimension(getDimensionAttr().getValue()); auto opName = getOperationName(); opName.consume_front("gpu."); SmallString<8> resultName({opName, "_", dimStr}); setNameFn(getResult(),resultName); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::ThreadIdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::ThreadIdOpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::WaitOp definitions //===----------------------------------------------------------------------===// namespace detail { WaitOpGenericAdaptorBase::WaitOpGenericAdaptorBase(WaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair WaitOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail WaitOpAdaptor::WaitOpAdaptor(WaitOp op) : WaitOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WaitOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair WaitOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange WaitOp::getAsyncDependenciesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair WaitOp::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult WaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute WaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WaitOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional WaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void WaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void WaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult WaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void WaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { odsState.addOperands(asyncDependencies); if (asyncToken) odsState.addTypes(asyncToken); } WaitOp WaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(asyncToken), std::forward(asyncDependencies)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WaitOp WaitOp::create(::mlir::ImplicitLocOpBuilder &builder, /*optional*/::mlir::Type asyncToken, ::mlir::ValueRange asyncDependencies) { return create(builder, builder.getLoc(), std::forward(asyncToken), std::forward(asyncDependencies)); } void WaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); odsState.addTypes(resultTypes); } WaitOp WaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WaitOp WaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void WaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); odsState.addTypes(resultTypes); } WaitOp WaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WaitOp WaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult WaitOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); if (valueGroup0.size() > 1) { return emitOpError("result group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup0.size(); } for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps5(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult WaitOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult WaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::Type, 1> asyncTokenTypes; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> asyncDependenciesOperands; ::llvm::SMLoc asyncDependenciesOperandsLoc; (void)asyncDependenciesOperandsLoc; { ::mlir::Type asyncTokenType; asyncDependenciesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseAsyncDependencies(parser, asyncTokenType, asyncDependenciesOperands); if (odsResult) return ::mlir::failure(); if (asyncTokenType) asyncTokenTypes.push_back(asyncTokenType); } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = mlir::gpu::AsyncTokenType::get(parser.getBuilder().getContext()); result.addTypes(asyncTokenTypes); if (parser.resolveOperands(asyncDependenciesOperands, odsBuildableType0, asyncDependenciesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void WaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; printAsyncDependencies(_odsPrinter, *this, (getAsyncToken() ? getAsyncToken().getType() : ::mlir::Type()), getAsyncDependencies()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::WaitOp) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::WarpExecuteOnLane0Op definitions //===----------------------------------------------------------------------===// namespace detail { WarpExecuteOnLane0OpGenericAdaptorBase::WarpExecuteOnLane0OpGenericAdaptorBase(WarpExecuteOnLane0Op op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair WarpExecuteOnLane0OpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } uint64_t WarpExecuteOnLane0OpGenericAdaptorBase::getWarpSize() { auto attr = getWarpSizeAttr(); return attr.getValue().getZExtValue(); } } // namespace detail WarpExecuteOnLane0OpAdaptor::WarpExecuteOnLane0OpAdaptor(WarpExecuteOnLane0Op op) : WarpExecuteOnLane0OpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult WarpExecuteOnLane0OpAdaptor::verify(::mlir::Location loc) { auto tblgen_warp_size = getProperties().warp_size; (void)tblgen_warp_size; if (!tblgen_warp_size) return emitError(loc, "'gpu.warp_execute_on_lane_0' op requires attribute 'warp_size'"); if (tblgen_warp_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_warp_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_warp_size).getType().isSignlessInteger(64))))) return emitError(loc, "'gpu.warp_execute_on_lane_0' op attribute 'warp_size' failed to satisfy constraint: 64-bit signless integer attribute"); return ::mlir::success(); } std::pair WarpExecuteOnLane0Op::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange WarpExecuteOnLane0Op::getArgsMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } std::pair WarpExecuteOnLane0Op::getODSResultIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumResults() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::LogicalResult WarpExecuteOnLane0Op::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.warp_size; auto attr = dict.get("warp_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `warp_size` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute WarpExecuteOnLane0Op::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.warp_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("warp_size", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code WarpExecuteOnLane0Op::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.warp_size.getAsOpaquePointer())); } std::optional WarpExecuteOnLane0Op::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "warp_size") return prop.warp_size; return std::nullopt; } void WarpExecuteOnLane0Op::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "warp_size") { prop.warp_size = ::llvm::dyn_cast_or_null>(value); return; } } void WarpExecuteOnLane0Op::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.warp_size) attrs.append("warp_size", prop.warp_size); } ::llvm::LogicalResult WarpExecuteOnLane0Op::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getWarpSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps27(attr, "warp_size", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult WarpExecuteOnLane0Op::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.warp_size))) return ::mlir::failure(); return ::mlir::success(); } void WarpExecuteOnLane0Op::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.warp_size); } uint64_t WarpExecuteOnLane0Op::getWarpSize() { auto attr = getWarpSizeAttr(); return attr.getValue().getZExtValue(); } void WarpExecuteOnLane0Op::setWarpSize(uint64_t attrValue) { getProperties().warp_size = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), attrValue); } WarpExecuteOnLane0Op WarpExecuteOnLane0Op::create(::mlir::OpBuilder &builder, ::mlir::Location location, TypeRange resultTypes, Value laneid, int64_t warpSize) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(laneid), std::forward(warpSize)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpExecuteOnLane0Op WarpExecuteOnLane0Op::create(::mlir::ImplicitLocOpBuilder &builder, TypeRange resultTypes, Value laneid, int64_t warpSize) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(laneid), std::forward(warpSize)); } WarpExecuteOnLane0Op WarpExecuteOnLane0Op::create(::mlir::OpBuilder &builder, ::mlir::Location location, TypeRange resultTypes, Value laneid, int64_t warpSize, ValueRange args, TypeRange blockArgTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(laneid), std::forward(warpSize), std::forward(args), std::forward(blockArgTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } WarpExecuteOnLane0Op WarpExecuteOnLane0Op::create(::mlir::ImplicitLocOpBuilder &builder, TypeRange resultTypes, Value laneid, int64_t warpSize, ValueRange args, TypeRange blockArgTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(laneid), std::forward(warpSize), std::forward(args), std::forward(blockArgTypes)); } ::llvm::LogicalResult WarpExecuteOnLane0Op::verifyInvariantsImpl() { auto tblgen_warp_size = getProperties().warp_size; (void)tblgen_warp_size; if (!tblgen_warp_size) return emitOpError("requires attribute 'warp_size'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_GPUOps27(*this, tblgen_warp_size, "warp_size"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps16(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; for (auto ®ion : ::llvm::MutableArrayRef((*this)->getRegion(0))) if (::mlir::failed(__mlir_ods_local_region_constraint_GPUOps2(*this, region, "warpRegion", index++))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult WarpExecuteOnLane0Op::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::llvm::LogicalResult WarpExecuteOnLane0Op::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.warp_size; auto warp_sizeAttrName = ::mlir::StringAttr::get(ctx, "warp_size"); auto attr = dict.get(warp_sizeAttrName); usedKeys.insert(warp_sizeAttrName); if (attr || /*isRequired=*/true) { if (!attr) { emitError() << "expected key entry for warp_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `warp_size` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::WarpExecuteOnLane0Op) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::detail::WarpExecuteOnLane0OpGenericAdaptorBase::Properties) namespace mlir::gpu { //===----------------------------------------------------------------------===// // ::mlir::gpu::YieldOp definitions //===----------------------------------------------------------------------===// namespace detail { YieldOpGenericAdaptorBase::YieldOpGenericAdaptorBase(YieldOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair YieldOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } } // namespace detail YieldOpAdaptor::YieldOpAdaptor(YieldOp op) : YieldOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult YieldOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair YieldOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 0) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange YieldOp::getValuesMutable() { auto range = getODSOperandIndexAndLength(0); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult YieldOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute YieldOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code YieldOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional YieldOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void YieldOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void YieldOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult YieldOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void YieldOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState) { /* nothing to do */ } YieldOp YieldOp::create(::mlir::OpBuilder &builder, ::mlir::Location location) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } YieldOp YieldOp::create(::mlir::ImplicitLocOpBuilder &builder) { return create(builder, builder.getLoc()); } void YieldOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange values) { odsState.addOperands(values); } YieldOp YieldOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange values) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(values)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } YieldOp YieldOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange values) { return create(builder, builder.getLoc(), std::forward(values)); } void YieldOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } YieldOp YieldOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } YieldOp YieldOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void YieldOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } YieldOp YieldOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } YieldOp YieldOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult YieldOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_GPUOps16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult YieldOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult YieldOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> valuesOperands; ::llvm::SMLoc valuesOperandsLoc; (void)valuesOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> valuesTypes; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } valuesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(valuesOperands)) return ::mlir::failure(); if (!valuesOperands.empty()) { if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(valuesTypes)) return ::mlir::failure(); } if (parser.resolveOperands(valuesOperands, valuesTypes, valuesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void YieldOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); if (!getValues().empty()) { _odsPrinter << ' '; _odsPrinter << getValues(); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getValues().getTypes(); } } void YieldOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::gpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::YieldOp) #endif // GET_OP_CLASSES