11471 lines
560 KiB
C++
11471 lines
560 KiB
C++
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
|
|
|* *|
|
|
|* Op Definitions *|
|
|
|* *|
|
|
|* Automatically generated file, do not edit! *|
|
|
|* From: XeGPU.td *|
|
|
|* *|
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
#ifdef GET_OP_LIST
|
|
#undef GET_OP_LIST
|
|
|
|
::mlir::xegpu::AllocNbarrierOp,
|
|
::mlir::xegpu::AtomicRMWOp,
|
|
::mlir::xegpu::ConvertLayoutOp,
|
|
::mlir::xegpu::CreateMemDescOp,
|
|
::mlir::xegpu::CreateNdDescOp,
|
|
::mlir::xegpu::DpasMxOp,
|
|
::mlir::xegpu::DpasOp,
|
|
::mlir::xegpu::FenceOp,
|
|
::mlir::xegpu::InitNbarrierOp,
|
|
::mlir::xegpu::LoadGatherOp,
|
|
::mlir::xegpu::LoadMatrixOp,
|
|
::mlir::xegpu::LoadNdOp,
|
|
::mlir::xegpu::NbarrierArriveOp,
|
|
::mlir::xegpu::NbarrierWaitOp,
|
|
::mlir::xegpu::PrefetchNdOp,
|
|
::mlir::xegpu::PrefetchOp,
|
|
::mlir::xegpu::StoreMatrixOp,
|
|
::mlir::xegpu::StoreNdOp,
|
|
::mlir::xegpu::StoreScatterOp,
|
|
::mlir::xegpu::TruncfOp
|
|
#endif // GET_OP_LIST
|
|
|
|
#ifdef GET_OP_CLASSES
|
|
#undef GET_OP_CLASSES
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Local Utility Method Definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU1(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::xegpu::TensorDescType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be TensorDesc describing regions of interested data., but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU2(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(1)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of 1-bit signless integer values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU3(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 8)))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4/5/6/7/8, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU4(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return ((((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType))))) || ((::llvm::isa<::mlir::IndexType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 8))))) || ((((type.isSignlessInteger(1))) || ((type.isSignlessInteger(4))) || ((type.isSignlessInteger(8))) || ((type.isSignlessInteger(16))) || ((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignedInteger(1))) || ((type.isSignedInteger(8))) || ((type.isSignedInteger(16))) || ((type.isSignedInteger(32))) || ((type.isSignedInteger(64))) || ((type.isUnsignedInteger(1))) || ((type.isUnsignedInteger(8))) || ((type.isUnsignedInteger(16))) || ((type.isUnsignedInteger(32))) || ((type.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(type))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(type))) || ((::llvm::isa<::mlir::Float8E5M2Type>(type))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || ((type.isF16())) || ((type.isF32())) || ((type.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(type))) || ((::llvm::isa<::mlir::FloatTF32Type>(type))))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type or index values of ranks 1/2/3/4/5/6/7/8 or 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU5(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank()
|
|
== 1)))) && ((::llvm::cast<::mlir::ShapedType>(type).hasStaticShape())) && ((XeGPUDialect::isSharedMemory(llvm::cast<mlir::MemRefType>(type))))) || (((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank()
|
|
== 2)))) && ((::llvm::cast<::mlir::ShapedType>(type).hasStaticShape())) && ((XeGPUDialect::isSharedMemory(llvm::cast<mlir::MemRefType>(type))))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be reside in share memory and statically 1d shaped memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or reside in share memory and statically 2d shaped memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values , but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU6(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::xegpu::MemDescType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be MemDesc describing the data in SLM, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU7(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() >= 1)))) || (((type.isUnsignedInteger(64))) || ((type.isUnsignedInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignlessInteger(32)))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be non-0-ranked.memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or 64-bit unsigned integer or 32-bit unsigned integer or 64-bit signless integer or 32-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU8(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::IndexType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be variadic of index, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU9(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 4)))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU10(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || (((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::Float8E8M0FNUType>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 4))))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be f8E8M0FNU type or vector of f8E8M0FNU type values of ranks 1/2/3/4, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU11(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((type.isSignlessInteger(8)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be 8-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU12(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((::llvm::isa<::mlir::xegpu::NbarrierType>(type)))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be !xegpu.nbarrier a custom XeGPU type representing a barrier., but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU13(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank()
|
|
== 1)))) || (((type.isUnsignedInteger(64))) || ((type.isUnsignedInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignlessInteger(32)))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be 1D memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or 64-bit unsigned integer or 32-bit unsigned integer or 64-bit signless integer or 32-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU14(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::IndexType>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((::llvm::isa<::mlir::IndexType>(type))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of index values or index, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU15(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!(((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(1)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((type.isSignlessInteger(1))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of 1-bit signless integer values or 1-bit signless integer, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU16(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank()
|
|
== 8))))) || ((((type.isSignlessInteger(1))) || ((type.isSignlessInteger(4))) || ((type.isSignlessInteger(8))) || ((type.isSignlessInteger(16))) || ((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignedInteger(1))) || ((type.isSignedInteger(8))) || ((type.isSignedInteger(16))) || ((type.isSignedInteger(32))) || ((type.isSignedInteger(64))) || ((type.isUnsignedInteger(1))) || ((type.isUnsignedInteger(8))) || ((type.isUnsignedInteger(16))) || ((type.isUnsignedInteger(32))) || ((type.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(type))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(type))) || ((::llvm::isa<::mlir::Float8E5M2Type>(type))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || ((type.isF16())) || ((type.isF32())) || ((type.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(type))) || ((::llvm::isa<::mlir::FloatTF32Type>(type))))))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4/5/6/7/8 or 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU17(
|
|
::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind,
|
|
unsigned valueIndex) {
|
|
if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return ((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) {
|
|
return op->emitOpError(valueKind) << " #" << valueIndex
|
|
<< " must be fixed-length vector of f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values, but got " << type;
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU1(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64)))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: 64-bit signless integer attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU1(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU1(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU2(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::arith::AtomicRMWKindAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: allowed 64-bit signless integer cases: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU2(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU2(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU3(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: DistributeLayoutAttr instance";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU3(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU3(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU4(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: i64 dense array attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU4(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU4(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU5(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::xegpu::MemorySpaceAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: Describe the location of data described by a `TensorDesc`:\n Global device memory (`Global`) or Shared local memory (`SLM`).";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU5(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU5(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU6(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::xegpu::FenceScopeAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: Describes the scope of fence.\n \"workgroup\" means that the scope is within each work group.\n \"gpu\" means the scope is across work groups within the gpu.";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU6(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU6(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU7(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU7(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU7(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU8(
|
|
::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
if (attr && !((::llvm::isa<::mlir::UnitAttr>(attr))))
|
|
return emitError() << "attribute '" << attrName
|
|
<< "' failed to satisfy constraint: unit attribute";
|
|
return ::mlir::success();
|
|
}
|
|
static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU8(
|
|
::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
|
|
return __mlir_ods_local_attr_constraint_XeGPU8(attr, attrName, [op]() {
|
|
return op->emitOpError();
|
|
});
|
|
}
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::AllocNbarrierOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
AllocNbarrierOpGenericAdaptorBase::AllocNbarrierOpGenericAdaptorBase(AllocNbarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
uint64_t AllocNbarrierOpGenericAdaptorBase::getNbarrierNum() {
|
|
auto attr = getNbarrierNumAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
AllocNbarrierOpAdaptor::AllocNbarrierOpAdaptor(AllocNbarrierOp op) : AllocNbarrierOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_nbarrier_num = getProperties().nbarrier_num; (void)tblgen_nbarrier_num;
|
|
if (!tblgen_nbarrier_num) return emitError(loc, "'xegpu.alloc_nbarrier' op requires attribute 'nbarrier_num'");
|
|
|
|
if (tblgen_nbarrier_num && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_nbarrier_num))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_nbarrier_num).getType().isSignlessInteger(64)))))
|
|
return emitError(loc, "'xegpu.alloc_nbarrier' op attribute 'nbarrier_num' failed to satisfy constraint: 64-bit signless integer attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.nbarrier_num;
|
|
auto attr = dict.get("nbarrier_num");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `nbarrier_num` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute AllocNbarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.nbarrier_num;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("nbarrier_num",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code AllocNbarrierOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.nbarrier_num.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> AllocNbarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "nbarrier_num")
|
|
return prop.nbarrier_num;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void AllocNbarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "nbarrier_num") {
|
|
prop.nbarrier_num = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.nbarrier_num)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void AllocNbarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.nbarrier_num) attrs.append("nbarrier_num", prop.nbarrier_num);
|
|
}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getNbarrierNumAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "nbarrier_num", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.nbarrier_num)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void AllocNbarrierOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.nbarrier_num);
|
|
}
|
|
|
|
uint64_t AllocNbarrierOp::getNbarrierNum() {
|
|
auto attr = getNbarrierNumAttr();
|
|
return attr.getValue().getZExtValue();
|
|
}
|
|
|
|
void AllocNbarrierOp::setNbarrierNum(uint64_t attrValue) {
|
|
getProperties().nbarrier_num = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), attrValue);
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::IntegerAttr nbarrier_num) {
|
|
odsState.getOrAddProperties<Properties>().nbarrier_num = nbarrier_num;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::IntegerAttr nbarrier_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::IntegerAttr nbarrier_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) {
|
|
odsState.getOrAddProperties<Properties>().nbarrier_num = nbarrier_num;
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, uint64_t nbarrier_num) {
|
|
odsState.getOrAddProperties<Properties>().nbarrier_num = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), nbarrier_num);
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, uint64_t nbarrier_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, uint64_t nbarrier_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) {
|
|
odsState.getOrAddProperties<Properties>().nbarrier_num = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), nbarrier_num);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_num)>(nbarrier_num));
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<AllocNbarrierOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void AllocNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<AllocNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOp::verifyInvariantsImpl() {
|
|
auto tblgen_nbarrier_num = getProperties().nbarrier_num; (void)tblgen_nbarrier_num;
|
|
if (!tblgen_nbarrier_num) return emitOpError("requires attribute 'nbarrier_num'");
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_nbarrier_num, "nbarrier_num")))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AllocNbarrierOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult AllocNbarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::IntegerAttr nbarrier_numAttr;
|
|
|
|
if (parser.parseCustomAttributeWithFallback(nbarrier_numAttr, parser.getBuilder().getIntegerType(64))) {
|
|
return ::mlir::failure();
|
|
}
|
|
if (nbarrier_numAttr) result.getOrAddProperties<AllocNbarrierOp::Properties>().nbarrier_num = nbarrier_numAttr;
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void AllocNbarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter.printAttributeWithoutType(getNbarrierNumAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("nbarrier_num");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::AllocNbarrierOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::AllocNbarrierOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::AtomicRMWOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
AtomicRMWOpGenericAdaptorBase::AtomicRMWOpGenericAdaptorBase(AtomicRMWOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::arith::AtomicRMWKind AtomicRMWOpGenericAdaptorBase::getKind() {
|
|
auto attr = getKindAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> AtomicRMWOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
AtomicRMWOpAdaptor::AtomicRMWOpAdaptor(AtomicRMWOp op) : AtomicRMWOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult AtomicRMWOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_kind = getProperties().kind; (void)tblgen_kind;
|
|
if (!tblgen_kind) return emitError(loc, "'xegpu.atomic_rmw' op requires attribute 'kind'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (tblgen_kind && !((::llvm::isa<::mlir::arith::AtomicRMWKindAttr>(tblgen_kind))))
|
|
return emitError(loc, "'xegpu.atomic_rmw' op attribute 'kind' failed to satisfy constraint: allowed 64-bit signless integer cases: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.atomic_rmw' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AtomicRMWOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.kind;
|
|
auto attr = dict.get("kind");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `kind` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute AtomicRMWOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.kind;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("kind",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code AtomicRMWOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.kind.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> AtomicRMWOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "kind")
|
|
return prop.kind;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void AtomicRMWOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "kind") {
|
|
prop.kind = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.kind)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void AtomicRMWOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.kind) attrs.append("kind", prop.kind);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
}
|
|
|
|
::llvm::LogicalResult AtomicRMWOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getKindAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU2(attr, "kind", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AtomicRMWOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.kind)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void AtomicRMWOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.kind);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
}
|
|
|
|
::mlir::arith::AtomicRMWKind AtomicRMWOp::getKind() {
|
|
auto attr = getKindAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> AtomicRMWOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void AtomicRMWOp::setKind(::mlir::arith::AtomicRMWKind attrValue) {
|
|
getProperties().kind = ::mlir::arith::AtomicRMWKindAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue);
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(tensorDesc);
|
|
odsState.addOperands(mask);
|
|
odsState.addOperands(value);
|
|
odsState.getOrAddProperties<Properties>().kind = kind;
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(tensorDesc);
|
|
odsState.addOperands(mask);
|
|
odsState.addOperands(value);
|
|
odsState.getOrAddProperties<Properties>().kind = kind;
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(tensorDesc);
|
|
odsState.addOperands(mask);
|
|
odsState.addOperands(value);
|
|
odsState.getOrAddProperties<Properties>().kind = ::mlir::arith::AtomicRMWKindAttr::get(odsBuilder.getContext(), kind);
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(tensorDesc);
|
|
odsState.addOperands(mask);
|
|
odsState.addOperands(value);
|
|
odsState.getOrAddProperties<Properties>().kind = ::mlir::arith::AtomicRMWKindAttr::get(odsBuilder.getContext(), kind);
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(kind)>(kind), std::forward<decltype(tensorDesc)>(tensorDesc), std::forward<decltype(mask)>(mask), std::forward<decltype(value)>(value), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<AtomicRMWOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void AtomicRMWOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<AtomicRMWOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult AtomicRMWOp::verifyInvariantsImpl() {
|
|
auto tblgen_kind = getProperties().kind; (void)tblgen_kind;
|
|
if (!tblgen_kind) return emitOpError("requires attribute 'kind'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU2(*this, tblgen_kind, "kind")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU2(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin()))) && (getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))))))
|
|
return emitOpError("failed to verify that all of {tensorDesc, value, result} have same element type");
|
|
if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(2).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(2).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()))))
|
|
return emitOpError("failed to verify that all of {tensorDesc, value, result} have same shape");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult AtomicRMWOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult AtomicRMWOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::arith::AtomicRMWKindAttr kindAttr;
|
|
::mlir::OpAsmParser::UnresolvedOperand tensorDescRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorDescOperands(&tensorDescRawOperand, 1); ::llvm::SMLoc tensorDescOperandsLoc;
|
|
(void)tensorDescOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc;
|
|
(void)maskOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc;
|
|
(void)valueOperandsLoc;
|
|
::mlir::Type tensorDescRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> tensorDescTypes(&tensorDescRawType, 1);
|
|
::mlir::Type maskRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> maskTypes(&maskRawType, 1);
|
|
::mlir::Type valueRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1);
|
|
::mlir::Type resultRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1);
|
|
|
|
{
|
|
::llvm::StringRef attrStr;
|
|
::mlir::NamedAttrList attrStorage;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (parser.parseOptionalKeyword(&attrStr, {"addf","addi","andi","assign","maximumf","maxnumf","maxs","maxu","minimumf","minnumf","mins","minu","mulf","muli","ori","xori"})) {
|
|
::mlir::StringAttr attrVal;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalAttribute(attrVal,
|
|
parser.getBuilder().getNoneType(),
|
|
"kind", attrStorage);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
attrStr = attrVal.getValue();
|
|
} else {
|
|
return parser.emitError(loc, "expected string or keyword containing one of the following enum values for attribute 'kind' [addf, addi, andi, assign, maximumf, maxnumf, maxs, maxu, minimumf, minnumf, mins, minu, mulf, muli, ori, xori]");
|
|
}
|
|
}
|
|
if (!attrStr.empty()) {
|
|
auto attrOptional = ::mlir::arith::symbolizeAtomicRMWKind(attrStr);
|
|
if (!attrOptional)
|
|
return parser.emitError(loc, "invalid ")
|
|
<< "kind attribute specification: \"" << attrStr << '"';;
|
|
|
|
kindAttr = ::mlir::arith::AtomicRMWKindAttr::get(parser.getBuilder().getContext(), *attrOptional);
|
|
result.getOrAddProperties<AtomicRMWOp::Properties>().kind = kindAttr;
|
|
}
|
|
}
|
|
|
|
tensorDescOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(tensorDescRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
maskOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(maskRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
valueOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(valueRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(tensorDescRawType))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
maskRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
valueRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
resultRawType = type;
|
|
}
|
|
result.addTypes(resultTypes);
|
|
if (parser.resolveOperands(tensorDescOperands, tensorDescTypes, tensorDescOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(maskOperands, maskTypes, maskOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void AtomicRMWOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
|
|
{
|
|
auto caseValue = getKind();
|
|
auto caseValueStr = stringifyAtomicRMWKind(caseValue);
|
|
_odsPrinter << caseValueStr;
|
|
}
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMask();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getValue();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("kind");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc().getType();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getMask().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getValue().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getResult().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void AtomicRMWOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::AtomicRMWOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::AtomicRMWOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::ConvertLayoutOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
ConvertLayoutOpGenericAdaptorBase::ConvertLayoutOpGenericAdaptorBase(ConvertLayoutOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOpGenericAdaptorBase::getInputLayout() {
|
|
auto attr = getInputLayoutAttr();
|
|
return attr;
|
|
}
|
|
|
|
::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOpGenericAdaptorBase::getTargetLayout() {
|
|
auto attr = getTargetLayoutAttr();
|
|
return attr;
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
ConvertLayoutOpAdaptor::ConvertLayoutOpAdaptor(ConvertLayoutOp op) : ConvertLayoutOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_input_layout = getProperties().input_layout; (void)tblgen_input_layout;
|
|
if (!tblgen_input_layout) return emitError(loc, "'xegpu.convert_layout' op requires attribute 'input_layout'");
|
|
auto tblgen_target_layout = getProperties().target_layout; (void)tblgen_target_layout;
|
|
if (!tblgen_target_layout) return emitError(loc, "'xegpu.convert_layout' op requires attribute 'target_layout'");
|
|
|
|
if (tblgen_input_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_input_layout))))
|
|
return emitError(loc, "'xegpu.convert_layout' op attribute 'input_layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_target_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_target_layout))))
|
|
return emitError(loc, "'xegpu.convert_layout' op attribute 'target_layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.input_layout;
|
|
auto attr = dict.get("input_layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `input_layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.target_layout;
|
|
auto attr = dict.get("target_layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `target_layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute ConvertLayoutOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.input_layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("input_layout",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.target_layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("target_layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code ConvertLayoutOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.input_layout.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.target_layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> ConvertLayoutOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "input_layout")
|
|
return prop.input_layout;
|
|
|
|
if (name == "target_layout")
|
|
return prop.target_layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void ConvertLayoutOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "input_layout") {
|
|
prop.input_layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.input_layout)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "target_layout") {
|
|
prop.target_layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.target_layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void ConvertLayoutOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.input_layout) attrs.append("input_layout", prop.input_layout);
|
|
|
|
if (prop.target_layout) attrs.append("target_layout", prop.target_layout);
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getInputLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "input_layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTargetLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "target_layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.input_layout)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readAttribute(prop.target_layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void ConvertLayoutOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.input_layout);
|
|
writer.writeAttribute(prop.target_layout);
|
|
}
|
|
|
|
::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOp::getInputLayout() {
|
|
auto attr = getInputLayoutAttr();
|
|
return attr;
|
|
}
|
|
|
|
::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOp::getTargetLayout() {
|
|
auto attr = getTargetLayoutAttr();
|
|
return attr;
|
|
}
|
|
|
|
void ConvertLayoutOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
odsState.addOperands(source);
|
|
odsState.getOrAddProperties<Properties>().input_layout = input_layout;
|
|
odsState.getOrAddProperties<Properties>().target_layout = target_layout;
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(source)>(source), std::forward<decltype(input_layout)>(input_layout), std::forward<decltype(target_layout)>(target_layout));
|
|
auto __res__ = ::llvm::dyn_cast<ConvertLayoutOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(source)>(source), std::forward<decltype(input_layout)>(input_layout), std::forward<decltype(target_layout)>(target_layout));
|
|
}
|
|
|
|
void ConvertLayoutOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
odsState.addOperands(source);
|
|
odsState.getOrAddProperties<Properties>().input_layout = input_layout;
|
|
odsState.getOrAddProperties<Properties>().target_layout = target_layout;
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(input_layout)>(input_layout), std::forward<decltype(target_layout)>(target_layout));
|
|
auto __res__ = ::llvm::dyn_cast<ConvertLayoutOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(input_layout)>(input_layout), std::forward<decltype(target_layout)>(target_layout));
|
|
}
|
|
|
|
void ConvertLayoutOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<ConvertLayoutOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<ConvertLayoutOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void ConvertLayoutOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<ConvertLayoutOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::verifyInvariantsImpl() {
|
|
auto tblgen_input_layout = getProperties().input_layout; (void)tblgen_input_layout;
|
|
if (!tblgen_input_layout) return emitOpError("requires attribute 'input_layout'");
|
|
auto tblgen_target_layout = getProperties().target_layout; (void)tblgen_target_layout;
|
|
if (!tblgen_target_layout) return emitOpError("requires attribute 'target_layout'");
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_input_layout, "input_layout")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_target_layout, "target_layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU4(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU4(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType()))))
|
|
return emitOpError("failed to verify that all of {source, result} have same type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult ConvertLayoutOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::mlir::Type sourceRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1);
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::Type type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
sourceRawType = type;
|
|
}
|
|
result.addTypes(sourceTypes[0]);
|
|
if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult ConvertLayoutOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.input_layout;
|
|
auto input_layoutAttrName = ::mlir::StringAttr::get(ctx, "input_layout");
|
|
auto attr = dict.get(input_layoutAttrName);
|
|
usedKeys.insert(input_layoutAttrName);
|
|
if (attr || /*isRequired=*/true) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for input_layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `input_layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.target_layout;
|
|
auto target_layoutAttrName = ::mlir::StringAttr::get(ctx, "target_layout");
|
|
auto attr = dict.get(target_layoutAttrName);
|
|
usedKeys.insert(target_layoutAttrName);
|
|
if (attr || /*isRequired=*/true) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for target_layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `target_layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void ConvertLayoutOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSource().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void ConvertLayoutOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::ConvertLayoutOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::ConvertLayoutOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::CreateMemDescOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
CreateMemDescOpGenericAdaptorBase::CreateMemDescOpGenericAdaptorBase(CreateMemDescOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
CreateMemDescOpAdaptor::CreateMemDescOpAdaptor(CreateMemDescOp op) : CreateMemDescOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult CreateMemDescOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult CreateMemDescOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute CreateMemDescOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code CreateMemDescOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> CreateMemDescOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void CreateMemDescOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void CreateMemDescOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult CreateMemDescOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void CreateMemDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type mem_desc, ::mlir::Value source) {
|
|
odsState.addOperands(source);
|
|
odsState.addTypes(mem_desc);
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type mem_desc, ::mlir::Value source) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(source)>(source));
|
|
auto __res__ = ::llvm::dyn_cast<CreateMemDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type mem_desc, ::mlir::Value source) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(source)>(source));
|
|
}
|
|
|
|
void CreateMemDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
odsState.addOperands(source);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source));
|
|
auto __res__ = ::llvm::dyn_cast<CreateMemDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source));
|
|
}
|
|
|
|
void CreateMemDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<CreateMemDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void CreateMemDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<CreateMemDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult CreateMemDescOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU5(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((llvm::cast<mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getNumElements()*llvm::cast<mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getElementTypeBitWidth()) == (llvm::cast<mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getNumElements()*llvm::cast<mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getElementTypeBitWidth()) && (llvm::cast<mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getNumElements()*llvm::cast<mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getElementTypeBitWidth()) == (llvm::cast<mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getNumElements()*llvm::cast<mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getElementTypeBitWidth()))))
|
|
return emitOpError("failed to verify that all of {source, mem_desc} have same size in bits");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult CreateMemDescOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult CreateMemDescOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::mlir::Type sourceRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1);
|
|
::mlir::Type mem_descRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> mem_descTypes(&mem_descRawType, 1);
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::Type type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
sourceRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(mem_descRawType))
|
|
return ::mlir::failure();
|
|
result.addTypes(mem_descTypes);
|
|
if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult CreateMemDescOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void CreateMemDescOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSource().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMemDesc().getType();
|
|
}
|
|
|
|
void CreateMemDescOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::CreateMemDescOp)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::CreateNdDescOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
CreateNdDescOpGenericAdaptorBase::CreateNdDescOpGenericAdaptorBase(CreateNdDescOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> CreateNdDescOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> CreateNdDescOpGenericAdaptorBase::getConstShape() {
|
|
auto attr = getConstShapeAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> CreateNdDescOpGenericAdaptorBase::getConstStrides() {
|
|
auto attr = getConstStridesAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
CreateNdDescOpAdaptor::CreateNdDescOpAdaptor(CreateNdDescOp op) : CreateNdDescOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult CreateNdDescOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_shape = getProperties().const_shape; (void)tblgen_const_shape;
|
|
auto tblgen_const_strides = getProperties().const_strides; (void)tblgen_const_strides;
|
|
|
|
if (tblgen_const_shape && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_shape))))
|
|
return emitError(loc, "'xegpu.create_nd_tdesc' op attribute 'const_shape' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_const_strides && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_strides))))
|
|
return emitError(loc, "'xegpu.create_nd_tdesc' op attribute 'const_strides' failed to satisfy constraint: i64 dense array attribute");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> CreateNdDescOp::getODSOperandIndexAndLength(unsigned index) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::mlir::MutableOperandRange CreateNdDescOp::getShapeMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange CreateNdDescOp::getStridesMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult CreateNdDescOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_shape;
|
|
auto attr = dict.get("const_shape");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_shape` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_strides;
|
|
auto attr = dict.get("const_strides");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_strides` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
|
|
::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {
|
|
return convertFromAttribute(propStorage, propAttr, emitError);
|
|
};
|
|
auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");;
|
|
;
|
|
if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() {
|
|
return emitError() << "for `operandSegmentSizes`: ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute CreateNdDescOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_shape;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_shape",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.const_strides;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_strides",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.operandSegmentSizes;
|
|
auto attr = [&]() -> ::mlir::Attribute {
|
|
return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage);
|
|
}();
|
|
attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code CreateNdDescOp::computePropertiesHash(const Properties &prop) {
|
|
auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code {
|
|
using ::llvm::hash_value;
|
|
return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));;
|
|
};
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_shape.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.const_strides.getAsOpaquePointer()),
|
|
hash_operandSegmentSizes_(prop.operandSegmentSizes));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> CreateNdDescOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_shape")
|
|
return prop.const_shape;
|
|
|
|
if (name == "const_strides")
|
|
return prop.const_strides;
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }();
|
|
return std::nullopt;
|
|
}
|
|
|
|
void CreateNdDescOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_shape") {
|
|
prop.const_shape = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_shape)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "const_strides") {
|
|
prop.const_strides = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_strides)>>(value);
|
|
return;
|
|
}
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") {
|
|
auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value);
|
|
if (!arrAttr) return;
|
|
if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t))
|
|
return;
|
|
llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin());
|
|
return;
|
|
}
|
|
}
|
|
|
|
void CreateNdDescOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_shape) attrs.append("const_shape", prop.const_shape);
|
|
|
|
if (prop.const_strides) attrs.append("const_strides", prop.const_strides);
|
|
attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }());
|
|
}
|
|
|
|
::llvm::LogicalResult CreateNdDescOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstShapeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_shape", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstStridesAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_strides", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult CreateNdDescOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.const_shape)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.const_strides)))
|
|
return ::mlir::failure();
|
|
|
|
if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
::mlir::DenseI32ArrayAttr attr;
|
|
if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure();
|
|
if (attr.size() > static_cast<int64_t>(sizeof(propStorage) / sizeof(int32_t))) {
|
|
reader.emitError("size mismatch for operand/result_segment_size");
|
|
return ::mlir::failure();
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>(attr), propStorage.begin());
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
auto readProp = [&]() {
|
|
|
|
if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
return reader.readSparseArray(::llvm::MutableArrayRef(propStorage));
|
|
;
|
|
return ::mlir::success();
|
|
};
|
|
if (::mlir::failed(readProp()))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void CreateNdDescOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.const_shape);
|
|
|
|
writer.writeOptionalAttribute(prop.const_strides);
|
|
|
|
if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage));
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
|
|
if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
writer.writeSparseArray(::llvm::ArrayRef(propStorage));
|
|
;
|
|
}
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> CreateNdDescOp::getConstShape() {
|
|
auto attr = getConstShapeAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> CreateNdDescOp::getConstStrides() {
|
|
auto attr = getConstStridesAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void CreateNdDescOp::setConstShape(::std::optional<::llvm::ArrayRef<int64_t>> attrValue) {
|
|
auto &odsProp = getProperties().const_shape;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void CreateNdDescOp::setConstStrides(::std::optional<::llvm::ArrayRef<int64_t>> attrValue) {
|
|
auto &odsProp = getProperties().const_strides;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type tdesc, TypedValue<MemRefType> source) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(tdesc)>(tdesc), std::forward<decltype(source)>(source));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, Type tdesc, TypedValue<MemRefType> source) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(tdesc)>(tdesc), std::forward<decltype(source)>(source));
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type tdesc, Value source, llvm::ArrayRef<OpFoldResult> shape, llvm::ArrayRef<OpFoldResult> strides) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(tdesc)>(tdesc), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, Type tdesc, Value source, llvm::ArrayRef<OpFoldResult> shape, llvm::ArrayRef<OpFoldResult> strides) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(tdesc)>(tdesc), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides));
|
|
}
|
|
|
|
void CreateNdDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(shape);
|
|
odsState.addOperands(strides);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(shape.size()), static_cast<int32_t>(strides.size())}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
if (const_shape) {
|
|
odsState.getOrAddProperties<Properties>().const_shape = const_shape;
|
|
}
|
|
if (const_strides) {
|
|
odsState.getOrAddProperties<Properties>().const_strides = const_strides;
|
|
}
|
|
odsState.addTypes(TensorDesc);
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides), std::forward<decltype(const_shape)>(const_shape), std::forward<decltype(const_strides)>(const_strides));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides), std::forward<decltype(const_shape)>(const_shape), std::forward<decltype(const_strides)>(const_strides));
|
|
}
|
|
|
|
void CreateNdDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(shape);
|
|
odsState.addOperands(strides);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(shape.size()), static_cast<int32_t>(strides.size())}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
if (const_shape) {
|
|
odsState.getOrAddProperties<Properties>().const_shape = const_shape;
|
|
}
|
|
if (const_strides) {
|
|
odsState.getOrAddProperties<Properties>().const_strides = const_strides;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides), std::forward<decltype(const_shape)>(const_shape), std::forward<decltype(const_strides)>(const_strides));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(shape)>(shape), std::forward<decltype(strides)>(strides), std::forward<decltype(const_shape)>(const_shape), std::forward<decltype(const_strides)>(const_strides));
|
|
}
|
|
|
|
void CreateNdDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<CreateNdDescOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void CreateNdDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<CreateNdDescOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult CreateNdDescOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_shape = getProperties().const_shape; (void)tblgen_const_shape;
|
|
auto tblgen_const_strides = getProperties().const_strides; (void)tblgen_const_strides;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_shape, "const_shape")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_strides, "const_strides")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU7(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult CreateNdDescOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult CreateNdDescOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> shapeOperands;
|
|
::llvm::SMLoc shapeOperandsLoc;
|
|
(void)shapeOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_shapeAttr;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> stridesOperands;
|
|
::llvm::SMLoc stridesOperandsLoc;
|
|
(void)stridesOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_stridesAttr;
|
|
::mlir::Type sourceRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1);
|
|
::mlir::Type TensorDescRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1);
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
if (parser.parseKeyword("shape"))
|
|
return ::mlir::failure();
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
{
|
|
shapeOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, shapeOperands, const_shapeAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
if (const_shapeAttr)
|
|
result.getOrAddProperties<CreateNdDescOp::Properties>().const_shape = const_shapeAttr;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("strides"))
|
|
return ::mlir::failure();
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
{
|
|
stridesOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, stridesOperands, const_stridesAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
if (const_stridesAttr)
|
|
result.getOrAddProperties<CreateNdDescOp::Properties>().const_strides = const_stridesAttr;
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::Type type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
sourceRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(TensorDescRawType))
|
|
return ::mlir::failure();
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, static_cast<int32_t>(shapeOperands.size()), static_cast<int32_t>(stridesOperands.size())}), result.getOrAddProperties<CreateNdDescOp::Properties>().operandSegmentSizes.begin());
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(TensorDescTypes);
|
|
if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(shapeOperands, odsBuildableType0, shapeOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(stridesOperands, odsBuildableType0, stridesOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void CreateNdDescOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
if (((!getShape().empty()) || (getConstShapeAttr()))) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "shape";
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
printDynamicIndexList(_odsPrinter, *this, getShape(), getConstShapeAttr());
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "strides";
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
printDynamicIndexList(_odsPrinter, *this, getStrides(), getConstStridesAttr());
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("operandSegmentSizes");
|
|
elidedAttrs.push_back("const_shape");
|
|
elidedAttrs.push_back("const_strides");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getSource().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc().getType();
|
|
}
|
|
|
|
void CreateNdDescOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::CreateNdDescOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::CreateNdDescOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::DpasMxOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
DpasMxOpGenericAdaptorBase::DpasMxOpGenericAdaptorBase(DpasMxOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> DpasMxOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutA() {
|
|
auto attr = getLayoutAAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutB() {
|
|
auto attr = getLayoutBAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutCd() {
|
|
auto attr = getLayoutCdAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutAScale() {
|
|
auto attr = getLayoutAScaleAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutBScale() {
|
|
auto attr = getLayoutBScaleAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
DpasMxOpAdaptor::DpasMxOpAdaptor(DpasMxOp op) : DpasMxOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult DpasMxOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a;
|
|
auto tblgen_layout_a_scale = getProperties().layout_a_scale; (void)tblgen_layout_a_scale;
|
|
auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b;
|
|
auto tblgen_layout_b_scale = getProperties().layout_b_scale; (void)tblgen_layout_b_scale;
|
|
auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd;
|
|
|
|
if (tblgen_layout_a && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a))))
|
|
return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_a' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_b && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b))))
|
|
return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_b' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_cd && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_cd))))
|
|
return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_cd' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_a_scale && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a_scale))))
|
|
return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_a_scale' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_b_scale && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b_scale))))
|
|
return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_b_scale' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> DpasMxOp::getODSOperandIndexAndLength(unsigned index) {
|
|
::llvm::ArrayRef<int32_t> sizeAttr = getProperties().operandSegmentSizes;
|
|
|
|
unsigned start = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
start += sizeAttr[i];
|
|
return {start, sizeAttr[index]};
|
|
}
|
|
|
|
::mlir::MutableOperandRange DpasMxOp::getAccMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange DpasMxOp::getScaleAMutable() {
|
|
auto range = getODSOperandIndexAndLength(3);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::mlir::MutableOperandRange DpasMxOp::getScaleBMutable() {
|
|
auto range = getODSOperandIndexAndLength(4);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)}));
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult DpasMxOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_a;
|
|
auto attr = dict.get("layout_a");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_a` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_a_scale;
|
|
auto attr = dict.get("layout_a_scale");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_a_scale` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_b;
|
|
auto attr = dict.get("layout_b");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_b` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_b_scale;
|
|
auto attr = dict.get("layout_b_scale");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_b_scale` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_cd;
|
|
auto attr = dict.get("layout_cd");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_cd` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
|
|
::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {
|
|
return convertFromAttribute(propStorage, propAttr, emitError);
|
|
};
|
|
auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");;
|
|
;
|
|
if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() {
|
|
return emitError() << "for `operandSegmentSizes`: ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute DpasMxOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_a;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_a",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_a_scale;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_a_scale",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_b;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_b",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_b_scale;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_b_scale",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_cd;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_cd",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.operandSegmentSizes;
|
|
auto attr = [&]() -> ::mlir::Attribute {
|
|
return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage);
|
|
}();
|
|
attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code DpasMxOp::computePropertiesHash(const Properties &prop) {
|
|
auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code {
|
|
using ::llvm::hash_value;
|
|
return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));;
|
|
};
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.layout_a.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_a_scale.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_b.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_b_scale.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_cd.getAsOpaquePointer()),
|
|
hash_operandSegmentSizes_(prop.operandSegmentSizes));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> DpasMxOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "layout_a")
|
|
return prop.layout_a;
|
|
|
|
if (name == "layout_a_scale")
|
|
return prop.layout_a_scale;
|
|
|
|
if (name == "layout_b")
|
|
return prop.layout_b;
|
|
|
|
if (name == "layout_b_scale")
|
|
return prop.layout_b_scale;
|
|
|
|
if (name == "layout_cd")
|
|
return prop.layout_cd;
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }();
|
|
return std::nullopt;
|
|
}
|
|
|
|
void DpasMxOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "layout_a") {
|
|
prop.layout_a = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_a)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_a_scale") {
|
|
prop.layout_a_scale = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_a_scale)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_b") {
|
|
prop.layout_b = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_b)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_b_scale") {
|
|
prop.layout_b_scale = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_b_scale)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_cd") {
|
|
prop.layout_cd = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_cd)>>(value);
|
|
return;
|
|
}
|
|
if (name == "operand_segment_sizes" || name == "operandSegmentSizes") {
|
|
auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value);
|
|
if (!arrAttr) return;
|
|
if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t))
|
|
return;
|
|
llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin());
|
|
return;
|
|
}
|
|
}
|
|
|
|
void DpasMxOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.layout_a) attrs.append("layout_a", prop.layout_a);
|
|
|
|
if (prop.layout_a_scale) attrs.append("layout_a_scale", prop.layout_a_scale);
|
|
|
|
if (prop.layout_b) attrs.append("layout_b", prop.layout_b);
|
|
|
|
if (prop.layout_b_scale) attrs.append("layout_b_scale", prop.layout_b_scale);
|
|
|
|
if (prop.layout_cd) attrs.append("layout_cd", prop.layout_cd);
|
|
attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }());
|
|
}
|
|
|
|
::llvm::LogicalResult DpasMxOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAScaleAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a_scale", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutBAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutBScaleAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b_scale", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutCdAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_cd", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DpasMxOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a_scale)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b_scale)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_cd)))
|
|
return ::mlir::failure();
|
|
|
|
if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
::mlir::DenseI32ArrayAttr attr;
|
|
if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure();
|
|
if (attr.size() > static_cast<int64_t>(sizeof(propStorage) / sizeof(int32_t))) {
|
|
reader.emitError("size mismatch for operand/result_segment_size");
|
|
return ::mlir::failure();
|
|
}
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>(attr), propStorage.begin());
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
auto readProp = [&]() {
|
|
|
|
if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
return reader.readSparseArray(::llvm::MutableArrayRef(propStorage));
|
|
;
|
|
return ::mlir::success();
|
|
};
|
|
if (::mlir::failed(readProp()))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DpasMxOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.layout_a);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_a_scale);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_b);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_b_scale);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_cd);
|
|
|
|
if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) {
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage));
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.operandSegmentSizes;
|
|
|
|
if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6)
|
|
writer.writeSparseArray(::llvm::ArrayRef(propStorage));
|
|
;
|
|
}
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutA() {
|
|
auto attr = getLayoutAAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutB() {
|
|
auto attr = getLayoutBAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutCd() {
|
|
auto attr = getLayoutCdAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutAScale() {
|
|
auto attr = getLayoutAScaleAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutBScale() {
|
|
auto attr = getLayoutBScaleAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void DpasMxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
odsState.addOperands(a);
|
|
odsState.addOperands(b);
|
|
if (acc)
|
|
odsState.addOperands(acc);
|
|
if (scale_a)
|
|
odsState.addOperands(scale_a);
|
|
if (scale_b)
|
|
odsState.addOperands(scale_b);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, (acc ? 1 : 0), (scale_a ? 1 : 0), (scale_b ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
if (layout_a) {
|
|
odsState.getOrAddProperties<Properties>().layout_a = layout_a;
|
|
}
|
|
if (layout_b) {
|
|
odsState.getOrAddProperties<Properties>().layout_b = layout_b;
|
|
}
|
|
if (layout_cd) {
|
|
odsState.getOrAddProperties<Properties>().layout_cd = layout_cd;
|
|
}
|
|
if (layout_a_scale) {
|
|
odsState.getOrAddProperties<Properties>().layout_a_scale = layout_a_scale;
|
|
}
|
|
if (layout_b_scale) {
|
|
odsState.getOrAddProperties<Properties>().layout_b_scale = layout_b_scale;
|
|
}
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(a)>(a), std::forward<decltype(b)>(b), std::forward<decltype(acc)>(acc), std::forward<decltype(scale_a)>(scale_a), std::forward<decltype(scale_b)>(scale_b), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd), std::forward<decltype(layout_a_scale)>(layout_a_scale), std::forward<decltype(layout_b_scale)>(layout_b_scale));
|
|
auto __res__ = ::llvm::dyn_cast<DpasMxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(a)>(a), std::forward<decltype(b)>(b), std::forward<decltype(acc)>(acc), std::forward<decltype(scale_a)>(scale_a), std::forward<decltype(scale_b)>(scale_b), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd), std::forward<decltype(layout_a_scale)>(layout_a_scale), std::forward<decltype(layout_b_scale)>(layout_b_scale));
|
|
}
|
|
|
|
void DpasMxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
odsState.addOperands(a);
|
|
odsState.addOperands(b);
|
|
if (acc)
|
|
odsState.addOperands(acc);
|
|
if (scale_a)
|
|
odsState.addOperands(scale_a);
|
|
if (scale_b)
|
|
odsState.addOperands(scale_b);
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, (acc ? 1 : 0), (scale_a ? 1 : 0), (scale_b ? 1 : 0)}), odsState.getOrAddProperties<Properties>().operandSegmentSizes.begin());
|
|
if (layout_a) {
|
|
odsState.getOrAddProperties<Properties>().layout_a = layout_a;
|
|
}
|
|
if (layout_b) {
|
|
odsState.getOrAddProperties<Properties>().layout_b = layout_b;
|
|
}
|
|
if (layout_cd) {
|
|
odsState.getOrAddProperties<Properties>().layout_cd = layout_cd;
|
|
}
|
|
if (layout_a_scale) {
|
|
odsState.getOrAddProperties<Properties>().layout_a_scale = layout_a_scale;
|
|
}
|
|
if (layout_b_scale) {
|
|
odsState.getOrAddProperties<Properties>().layout_b_scale = layout_b_scale;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(a)>(a), std::forward<decltype(b)>(b), std::forward<decltype(acc)>(acc), std::forward<decltype(scale_a)>(scale_a), std::forward<decltype(scale_b)>(scale_b), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd), std::forward<decltype(layout_a_scale)>(layout_a_scale), std::forward<decltype(layout_b_scale)>(layout_b_scale));
|
|
auto __res__ = ::llvm::dyn_cast<DpasMxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(a)>(a), std::forward<decltype(b)>(b), std::forward<decltype(acc)>(acc), std::forward<decltype(scale_a)>(scale_a), std::forward<decltype(scale_b)>(scale_b), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd), std::forward<decltype(layout_a_scale)>(layout_a_scale), std::forward<decltype(layout_b_scale)>(layout_b_scale));
|
|
}
|
|
|
|
void DpasMxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<DpasMxOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DpasMxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DpasMxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DpasMxOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult DpasMxOp::verifyInvariantsImpl() {
|
|
auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a;
|
|
auto tblgen_layout_a_scale = getProperties().layout_a_scale; (void)tblgen_layout_a_scale;
|
|
auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b;
|
|
auto tblgen_layout_b_scale = getProperties().layout_b_scale; (void)tblgen_layout_b_scale;
|
|
auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a, "layout_a")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b, "layout_b")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_cd, "layout_cd")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a_scale, "layout_a_scale")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b_scale, "layout_b_scale")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
if (valueGroup2.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup2.size();
|
|
}
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
if (valueGroup3.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup3.size();
|
|
}
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU10(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup4 = getODSOperands(4);
|
|
|
|
if (valueGroup4.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup4.size();
|
|
}
|
|
|
|
for (auto v : valueGroup4) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU10(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DpasMxOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult DpasMxOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand aRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> aOperands(&aRawOperand, 1); ::llvm::SMLoc aOperandsLoc;
|
|
(void)aOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand bRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bOperands(&bRawOperand, 1); ::llvm::SMLoc bOperandsLoc;
|
|
(void)bOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands;
|
|
::llvm::SMLoc accOperandsLoc;
|
|
(void)accOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> scale_aOperands;
|
|
::llvm::SMLoc scale_aOperandsLoc;
|
|
(void)scale_aOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> scale_bOperands;
|
|
::llvm::SMLoc scale_bOperandsLoc;
|
|
(void)scale_bOperandsLoc;
|
|
::llvm::ArrayRef<::mlir::Type> allOperandTypes;
|
|
::llvm::ArrayRef<::mlir::Type> allResultTypes;
|
|
|
|
aOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(aRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
bOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(bRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
|
|
{
|
|
accOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
accOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
if (::mlir::succeeded(parser.parseOptionalKeyword("scale_a"))) {
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
scale_aOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
scale_aOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
if (::mlir::succeeded(parser.parseOptionalKeyword("scale_b"))) {
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
scale_bOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
scale_bOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
::mlir::FunctionType allOperand__allResult_functionType;
|
|
if (parser.parseType(allOperand__allResult_functionType))
|
|
return ::mlir::failure();
|
|
allOperandTypes = allOperand__allResult_functionType.getInputs();
|
|
allResultTypes = allOperand__allResult_functionType.getResults();
|
|
::llvm::copy(::llvm::ArrayRef<int32_t>({1, 1, static_cast<int32_t>(accOperands.size()), static_cast<int32_t>(scale_aOperands.size()), static_cast<int32_t>(scale_bOperands.size())}), result.getOrAddProperties<DpasMxOp::Properties>().operandSegmentSizes.begin());
|
|
result.addTypes(allResultTypes);
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(aOperands, bOperands, accOperands, scale_aOperands, scale_bOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DpasMxOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getA();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getB();
|
|
if (getAcc()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getAcc())
|
|
_odsPrinter << value;
|
|
}
|
|
if (getScaleA()) {
|
|
_odsPrinter << ' ' << "scale_a";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getScaleA())
|
|
_odsPrinter << value;
|
|
}
|
|
if (getScaleB()) {
|
|
_odsPrinter << ' ' << "scale_b";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getScaleB())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("operandSegmentSizes");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter.printFunctionalType(getOperation()->getOperandTypes(), getOperation()->getResultTypes());
|
|
}
|
|
|
|
void DpasMxOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::DpasMxOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::DpasMxOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::DpasOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
DpasOpGenericAdaptorBase::DpasOpGenericAdaptorBase(DpasOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> DpasOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutA() {
|
|
auto attr = getLayoutAAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutB() {
|
|
auto attr = getLayoutBAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutCd() {
|
|
auto attr = getLayoutCdAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
DpasOpAdaptor::DpasOpAdaptor(DpasOp op) : DpasOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult DpasOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a;
|
|
auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b;
|
|
auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd;
|
|
|
|
if (tblgen_layout_a && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a))))
|
|
return emitError(loc, "'xegpu.dpas' op attribute 'layout_a' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_b && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b))))
|
|
return emitError(loc, "'xegpu.dpas' op attribute 'layout_b' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
|
|
if (tblgen_layout_cd && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_cd))))
|
|
return emitError(loc, "'xegpu.dpas' op attribute 'layout_cd' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> DpasOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange DpasOp::getAccMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult DpasOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_a;
|
|
auto attr = dict.get("layout_a");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_a` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_b;
|
|
auto attr = dict.get("layout_b");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_b` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout_cd;
|
|
auto attr = dict.get("layout_cd");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout_cd` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute DpasOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_a;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_a",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_b;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_b",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout_cd;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout_cd",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code DpasOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.layout_a.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_b.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout_cd.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> DpasOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "layout_a")
|
|
return prop.layout_a;
|
|
|
|
if (name == "layout_b")
|
|
return prop.layout_b;
|
|
|
|
if (name == "layout_cd")
|
|
return prop.layout_cd;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void DpasOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "layout_a") {
|
|
prop.layout_a = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_a)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_b") {
|
|
prop.layout_b = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_b)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout_cd") {
|
|
prop.layout_cd = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout_cd)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void DpasOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.layout_a) attrs.append("layout_a", prop.layout_a);
|
|
|
|
if (prop.layout_b) attrs.append("layout_b", prop.layout_b);
|
|
|
|
if (prop.layout_cd) attrs.append("layout_cd", prop.layout_cd);
|
|
}
|
|
|
|
::llvm::LogicalResult DpasOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutBAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutCdAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_cd", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DpasOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout_cd)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DpasOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.layout_a);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_b);
|
|
|
|
writer.writeOptionalAttribute(prop.layout_cd);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutA() {
|
|
auto attr = getLayoutAAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutB() {
|
|
auto attr = getLayoutBAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutCd() {
|
|
auto attr = getLayoutCdAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void DpasOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
odsState.addOperands(lhs);
|
|
odsState.addOperands(rhs);
|
|
if (acc)
|
|
odsState.addOperands(acc);
|
|
if (layout_a) {
|
|
odsState.getOrAddProperties<Properties>().layout_a = layout_a;
|
|
}
|
|
if (layout_b) {
|
|
odsState.getOrAddProperties<Properties>().layout_b = layout_b;
|
|
}
|
|
if (layout_cd) {
|
|
odsState.getOrAddProperties<Properties>().layout_cd = layout_cd;
|
|
}
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(lhs)>(lhs), std::forward<decltype(rhs)>(rhs), std::forward<decltype(acc)>(acc), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd));
|
|
auto __res__ = ::llvm::dyn_cast<DpasOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(lhs)>(lhs), std::forward<decltype(rhs)>(rhs), std::forward<decltype(acc)>(acc), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd));
|
|
}
|
|
|
|
void DpasOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
odsState.addOperands(lhs);
|
|
odsState.addOperands(rhs);
|
|
if (acc)
|
|
odsState.addOperands(acc);
|
|
if (layout_a) {
|
|
odsState.getOrAddProperties<Properties>().layout_a = layout_a;
|
|
}
|
|
if (layout_b) {
|
|
odsState.getOrAddProperties<Properties>().layout_b = layout_b;
|
|
}
|
|
if (layout_cd) {
|
|
odsState.getOrAddProperties<Properties>().layout_cd = layout_cd;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(lhs)>(lhs), std::forward<decltype(rhs)>(rhs), std::forward<decltype(acc)>(acc), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd));
|
|
auto __res__ = ::llvm::dyn_cast<DpasOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(lhs)>(lhs), std::forward<decltype(rhs)>(rhs), std::forward<decltype(acc)>(acc), std::forward<decltype(layout_a)>(layout_a), std::forward<decltype(layout_b)>(layout_b), std::forward<decltype(layout_cd)>(layout_cd));
|
|
}
|
|
|
|
void DpasOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<DpasOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<DpasOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void DpasOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<DpasOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult DpasOp::verifyInvariantsImpl() {
|
|
auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a;
|
|
auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b;
|
|
auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a, "layout_a")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b, "layout_b")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_cd, "layout_cd")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
if (valueGroup2.size() > 1) {
|
|
return emitOpError("operand group starting at #") << index
|
|
<< " requires 0 or 1 element, but found " << valueGroup2.size();
|
|
}
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))))))
|
|
return emitOpError("failed to verify that all of {lhs, rhs} have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult DpasOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult DpasOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc;
|
|
(void)lhsOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc;
|
|
(void)rhsOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands;
|
|
::llvm::SMLoc accOperandsLoc;
|
|
(void)accOperandsLoc;
|
|
::mlir::Type lhsRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1);
|
|
::mlir::Type rhsRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1);
|
|
::llvm::SmallVector<::mlir::Type, 1> accTypes;
|
|
::mlir::Type resultRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1);
|
|
|
|
lhsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(lhsRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
rhsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(rhsRawOperand))
|
|
return ::mlir::failure();
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
|
|
{
|
|
accOperandsLoc = parser.getCurrentLocation();
|
|
::mlir::OpAsmParser::UnresolvedOperand operand;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalOperand(operand);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
accOperands.push_back(operand);
|
|
}
|
|
}
|
|
}
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
lhsRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
rhsRawType = type;
|
|
}
|
|
if (::mlir::succeeded(parser.parseOptionalComma())) {
|
|
|
|
{
|
|
::mlir::Type optionalType;
|
|
::mlir::OptionalParseResult parseResult =
|
|
parser.parseOptionalType(optionalType);
|
|
if (parseResult.has_value()) {
|
|
if (failed(*parseResult))
|
|
return ::mlir::failure();
|
|
accTypes.push_back(optionalType);
|
|
}
|
|
}
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
resultRawType = type;
|
|
}
|
|
result.addTypes(resultTypes);
|
|
if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (!accOperands.empty()) {
|
|
if (parser.resolveOperands(accOperands, accTypes, accOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void DpasOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getLhs();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getRhs();
|
|
if (getAcc()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
if (::mlir::Value value = getAcc())
|
|
_odsPrinter << value;
|
|
}
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getLhs().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getRhs().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
if (getAcc()) {
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << (getAcc() ? ::llvm::ArrayRef<::mlir::Type>(getAcc().getType()) : ::llvm::ArrayRef<::mlir::Type>());
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getResult().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void DpasOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::DpasOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::DpasOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::FenceOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
FenceOpGenericAdaptorBase::FenceOpGenericAdaptorBase(FenceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::mlir::xegpu::MemorySpace FenceOpGenericAdaptorBase::getMemoryKind() {
|
|
auto attr = getMemoryKindAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::mlir::xegpu::FenceScope FenceOpGenericAdaptorBase::getFenceScope() {
|
|
auto attr = getFenceScopeAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
FenceOpAdaptor::FenceOpAdaptor(FenceOp op) : FenceOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult FenceOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_fence_scope = getProperties().fence_scope; (void)tblgen_fence_scope;
|
|
if (!tblgen_fence_scope) return emitError(loc, "'xegpu.fence' op requires attribute 'fence_scope'");
|
|
auto tblgen_memory_kind = getProperties().memory_kind; (void)tblgen_memory_kind;
|
|
if (!tblgen_memory_kind) return emitError(loc, "'xegpu.fence' op requires attribute 'memory_kind'");
|
|
|
|
if (tblgen_memory_kind && !((::llvm::isa<::mlir::xegpu::MemorySpaceAttr>(tblgen_memory_kind))))
|
|
return emitError(loc, "'xegpu.fence' op attribute 'memory_kind' failed to satisfy constraint: Describe the location of data described by a `TensorDesc`:\n Global device memory (`Global`) or Shared local memory (`SLM`).");
|
|
|
|
if (tblgen_fence_scope && !((::llvm::isa<::mlir::xegpu::FenceScopeAttr>(tblgen_fence_scope))))
|
|
return emitError(loc, "'xegpu.fence' op attribute 'fence_scope' failed to satisfy constraint: Describes the scope of fence.\n \"workgroup\" means that the scope is within each work group.\n \"gpu\" means the scope is across work groups within the gpu.");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult FenceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.fence_scope;
|
|
auto attr = dict.get("fence_scope");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `fence_scope` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.memory_kind;
|
|
auto attr = dict.get("memory_kind");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `memory_kind` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute FenceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.fence_scope;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("fence_scope",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.memory_kind;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("memory_kind",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code FenceOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.fence_scope.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.memory_kind.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> FenceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "fence_scope")
|
|
return prop.fence_scope;
|
|
|
|
if (name == "memory_kind")
|
|
return prop.memory_kind;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void FenceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "fence_scope") {
|
|
prop.fence_scope = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.fence_scope)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "memory_kind") {
|
|
prop.memory_kind = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.memory_kind)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void FenceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.fence_scope) attrs.append("fence_scope", prop.fence_scope);
|
|
|
|
if (prop.memory_kind) attrs.append("memory_kind", prop.memory_kind);
|
|
}
|
|
|
|
::llvm::LogicalResult FenceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getFenceScopeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU6(attr, "fence_scope", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getMemoryKindAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU5(attr, "memory_kind", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult FenceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.fence_scope)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readAttribute(prop.memory_kind)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void FenceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.fence_scope);
|
|
writer.writeAttribute(prop.memory_kind);
|
|
}
|
|
|
|
::mlir::xegpu::MemorySpace FenceOp::getMemoryKind() {
|
|
auto attr = getMemoryKindAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
::mlir::xegpu::FenceScope FenceOp::getFenceScope() {
|
|
auto attr = getFenceScopeAttr();
|
|
return attr.getValue();
|
|
}
|
|
|
|
void FenceOp::setMemoryKind(::mlir::xegpu::MemorySpace attrValue) {
|
|
getProperties().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue);
|
|
}
|
|
|
|
void FenceOp::setFenceScope(::mlir::xegpu::FenceScope attrValue) {
|
|
getProperties().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue);
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
odsState.getOrAddProperties<Properties>().memory_kind = memory_kind;
|
|
odsState.getOrAddProperties<Properties>().fence_scope = fence_scope;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
odsState.getOrAddProperties<Properties>().memory_kind = memory_kind;
|
|
odsState.getOrAddProperties<Properties>().fence_scope = fence_scope;
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
odsState.getOrAddProperties<Properties>().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(odsBuilder.getContext(), memory_kind);
|
|
odsState.getOrAddProperties<Properties>().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(odsBuilder.getContext(), fence_scope);
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
odsState.getOrAddProperties<Properties>().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(odsBuilder.getContext(), memory_kind);
|
|
odsState.getOrAddProperties<Properties>().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(odsBuilder.getContext(), fence_scope);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(memory_kind)>(memory_kind), std::forward<decltype(fence_scope)>(fence_scope));
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<FenceOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void FenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 0u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<FenceOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult FenceOp::verifyInvariantsImpl() {
|
|
auto tblgen_fence_scope = getProperties().fence_scope; (void)tblgen_fence_scope;
|
|
if (!tblgen_fence_scope) return emitOpError("requires attribute 'fence_scope'");
|
|
auto tblgen_memory_kind = getProperties().memory_kind; (void)tblgen_memory_kind;
|
|
if (!tblgen_memory_kind) return emitOpError("requires attribute 'memory_kind'");
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU5(*this, tblgen_memory_kind, "memory_kind")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU6(*this, tblgen_fence_scope, "fence_scope")))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult FenceOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult FenceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::xegpu::MemorySpaceAttr memory_kindAttr;
|
|
::mlir::xegpu::FenceScopeAttr fence_scopeAttr;
|
|
if (parser.parseKeyword("memory_kind"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseCustomAttributeWithFallback(memory_kindAttr, ::mlir::Type{})) {
|
|
return ::mlir::failure();
|
|
}
|
|
if (memory_kindAttr) result.getOrAddProperties<FenceOp::Properties>().memory_kind = memory_kindAttr;
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
if (parser.parseKeyword("fence_scope"))
|
|
return ::mlir::failure();
|
|
if (parser.parseEqual())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseCustomAttributeWithFallback(fence_scopeAttr, ::mlir::Type{})) {
|
|
return ::mlir::failure();
|
|
}
|
|
if (fence_scopeAttr) result.getOrAddProperties<FenceOp::Properties>().fence_scope = fence_scopeAttr;
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void FenceOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ' << "memory_kind";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter.printStrippedAttrOrType(getMemoryKindAttr());
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ' << "fence_scope";
|
|
_odsPrinter << ' ' << "=";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter.printStrippedAttrOrType(getFenceScopeAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("memory_kind");
|
|
elidedAttrs.push_back("fence_scope");
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::FenceOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::FenceOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::InitNbarrierOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
InitNbarrierOpGenericAdaptorBase::InitNbarrierOpGenericAdaptorBase(InitNbarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
InitNbarrierOpAdaptor::InitNbarrierOpAdaptor(InitNbarrierOp op) : InitNbarrierOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult InitNbarrierOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult InitNbarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute InitNbarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code InitNbarrierOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> InitNbarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void InitNbarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void InitNbarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult InitNbarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void InitNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
odsState.addOperands(nbarrier_id);
|
|
odsState.addOperands(participant_thread_num);
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(nbarrier_id)>(nbarrier_id), std::forward<decltype(participant_thread_num)>(participant_thread_num));
|
|
auto __res__ = ::llvm::dyn_cast<InitNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(nbarrier_id)>(nbarrier_id), std::forward<decltype(participant_thread_num)>(participant_thread_num));
|
|
}
|
|
|
|
void InitNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
odsState.addOperands(nbarrier_id);
|
|
odsState.addOperands(participant_thread_num);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_id)>(nbarrier_id), std::forward<decltype(participant_thread_num)>(participant_thread_num));
|
|
auto __res__ = ::llvm::dyn_cast<InitNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier_id)>(nbarrier_id), std::forward<decltype(participant_thread_num)>(participant_thread_num));
|
|
}
|
|
|
|
void InitNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<InitNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void InitNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<InitNbarrierOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult InitNbarrierOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU11(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU11(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult InitNbarrierOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult InitNbarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand nbarrier_idRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrier_idOperands(&nbarrier_idRawOperand, 1); ::llvm::SMLoc nbarrier_idOperandsLoc;
|
|
(void)nbarrier_idOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand participant_thread_numRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> participant_thread_numOperands(&participant_thread_numRawOperand, 1); ::llvm::SMLoc participant_thread_numOperandsLoc;
|
|
(void)participant_thread_numOperandsLoc;
|
|
::mlir::Type nbarrier_idRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> nbarrier_idTypes(&nbarrier_idRawType, 1);
|
|
::mlir::Type participant_thread_numRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> participant_thread_numTypes(&participant_thread_numRawType, 1);
|
|
::mlir::Type resultRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1);
|
|
|
|
nbarrier_idOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(nbarrier_idRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
participant_thread_numOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(participant_thread_numRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::IntegerType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
nbarrier_idRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::IntegerType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
participant_thread_numRawType = type;
|
|
}
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(resultRawType))
|
|
return ::mlir::failure();
|
|
result.addTypes(resultTypes);
|
|
if (parser.resolveOperands(nbarrier_idOperands, nbarrier_idTypes, nbarrier_idOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(participant_thread_numOperands, participant_thread_numTypes, participant_thread_numOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void InitNbarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getNbarrierId();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getParticipantThreadNum();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getNbarrierId().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getParticipantThreadNum().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getResult().getType();
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::InitNbarrierOp)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::LoadGatherOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
LoadGatherOpGenericAdaptorBase::LoadGatherOpGenericAdaptorBase(LoadGatherOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::std::optional<uint64_t> LoadGatherOpGenericAdaptorBase::getChunkSize() {
|
|
auto attr = getChunkSizeAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadGatherOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
LoadGatherOpAdaptor::LoadGatherOpAdaptor(LoadGatherOp op) : LoadGatherOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult LoadGatherOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size;
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (tblgen_chunk_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_chunk_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_chunk_size).getType().isSignlessInteger(64)))))
|
|
return emitError(loc, "'xegpu.load' op attribute 'chunk_size' failed to satisfy constraint: 64-bit signless integer attribute");
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.load' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.load' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.load' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.load' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.chunk_size;
|
|
auto attr = dict.get("chunk_size");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute LoadGatherOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.chunk_size;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("chunk_size",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code LoadGatherOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.chunk_size.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> LoadGatherOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "chunk_size")
|
|
return prop.chunk_size;
|
|
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void LoadGatherOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "chunk_size") {
|
|
prop.chunk_size = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.chunk_size)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void LoadGatherOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.chunk_size) attrs.append("chunk_size", prop.chunk_size);
|
|
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getChunkSizeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "chunk_size", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.chunk_size)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadGatherOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.chunk_size);
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
}
|
|
|
|
::std::optional<uint64_t> LoadGatherOp::getChunkSize() {
|
|
auto attr = getChunkSizeAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadGatherOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void LoadGatherOp::setChunkSize(::std::optional<uint64_t> attrValue) {
|
|
auto &odsProp = getProperties().chunk_size;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadGatherOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadGatherOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadGatherOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value source, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value source, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint));
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value source, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value source, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadGatherOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(offsets);
|
|
odsState.addOperands(mask);
|
|
if (chunk_size) {
|
|
odsState.getOrAddProperties<Properties>().chunk_size = chunk_size;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(value);
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadGatherOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(offsets);
|
|
odsState.addOperands(mask);
|
|
if (chunk_size) {
|
|
odsState.getOrAddProperties<Properties>().chunk_size = chunk_size;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadGatherOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<LoadGatherOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void LoadGatherOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 3u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadGatherOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::verifyInvariantsImpl() {
|
|
auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size;
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_chunk_size, "chunk_size")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU15(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult LoadGatherOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc;
|
|
(void)maskOperandsLoc;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
::mlir::Type valueRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1);
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(offsetsRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
maskOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(maskRawOperand))
|
|
return ::mlir::failure();
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::Type type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
valueRawType = type;
|
|
}
|
|
result.addTypes(valueTypes);
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(sourceOperands, offsetsOperands, maskOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadGatherOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.chunk_size;
|
|
auto chunk_sizeAttrName = ::mlir::StringAttr::get(ctx, "chunk_size");
|
|
auto attr = dict.get(chunk_sizeAttrName);
|
|
usedKeys.insert(chunk_sizeAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for chunk_size in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadGatherOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getOffsets();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMask();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getValue().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void LoadGatherOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadGatherOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadGatherOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::LoadMatrixOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
LoadMatrixOpGenericAdaptorBase::LoadMatrixOpGenericAdaptorBase(LoadMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> LoadMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> LoadMatrixOpGenericAdaptorBase::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> LoadMatrixOpGenericAdaptorBase::getSubgroupBlockIo() {
|
|
auto attr = getSubgroupBlockIoAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadMatrixOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
LoadMatrixOpAdaptor::LoadMatrixOpAdaptor(LoadMatrixOp op) : LoadMatrixOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult LoadMatrixOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitError(loc, "'xegpu.load_matrix' op requires attribute 'const_offsets'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io;
|
|
|
|
if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets))))
|
|
return emitError(loc, "'xegpu.load_matrix' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_subgroup_block_io && !((::llvm::isa<::mlir::UnitAttr>(tblgen_subgroup_block_io))))
|
|
return emitError(loc, "'xegpu.load_matrix' op attribute 'subgroup_block_io' failed to satisfy constraint: unit attribute");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.load_matrix' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> LoadMatrixOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange LoadMatrixOp::getOffsetsMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_offsets;
|
|
auto attr = dict.get("const_offsets");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.subgroup_block_io;
|
|
auto attr = dict.get("subgroup_block_io");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute LoadMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_offsets;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_offsets",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.subgroup_block_io;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("subgroup_block_io",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code LoadMatrixOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_offsets.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.subgroup_block_io.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> LoadMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_offsets")
|
|
return prop.const_offsets;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
|
|
if (name == "subgroup_block_io")
|
|
return prop.subgroup_block_io;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void LoadMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_offsets") {
|
|
prop.const_offsets = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_offsets)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "subgroup_block_io") {
|
|
prop.subgroup_block_io = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.subgroup_block_io)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void LoadMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
|
|
if (prop.subgroup_block_io) attrs.append("subgroup_block_io", prop.subgroup_block_io);
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getSubgroupBlockIoAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "subgroup_block_io", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.const_offsets)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.subgroup_block_io)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.const_offsets);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
|
|
writer.writeOptionalAttribute(prop.subgroup_block_io);
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> LoadMatrixOp::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> LoadMatrixOp::getSubgroupBlockIo() {
|
|
auto attr = getSubgroupBlockIoAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadMatrixOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void LoadMatrixOp::setConstOffsets(::llvm::ArrayRef<int64_t> attrValue) {
|
|
getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue);
|
|
}
|
|
|
|
void LoadMatrixOp::setSubgroupBlockIo(bool attrValue) {
|
|
auto &odsProp = getProperties().subgroup_block_io;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type res, TypedValue<MemDescType> mem_desc, llvm::ArrayRef<OpFoldResult> offsets, DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, Type res, TypedValue<MemDescType> mem_desc, llvm::ArrayRef<OpFoldResult> offsets, DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(res);
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(res)>(res), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<LoadMatrixOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void LoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_subgroup_block_io, "subgroup_block_io")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin()))) && (getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))))))
|
|
return emitOpError("failed to verify that all of {mem_desc, res} have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult LoadMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand mem_descRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mem_descOperands(&mem_descRawOperand, 1); ::llvm::SMLoc mem_descOperandsLoc;
|
|
(void)mem_descOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands;
|
|
::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_offsetsAttr;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
::llvm::SmallVector<::mlir::Type, 1> allResultTypes;
|
|
|
|
mem_descOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mem_descRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
result.getOrAddProperties<LoadMatrixOp::Properties>().const_offsets = const_offsetsAttr;
|
|
}
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allResultTypes))
|
|
return ::mlir::failure();
|
|
result.addTypes(allResultTypes);
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(mem_descOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadMatrixOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.subgroup_block_io;
|
|
auto subgroup_block_ioAttrName = ::mlir::StringAttr::get(ctx, "subgroup_block_io");
|
|
auto attr = dict.get(subgroup_block_ioAttrName);
|
|
usedKeys.insert(subgroup_block_ioAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for subgroup_block_io in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMemDesc();
|
|
printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
elidedProps.push_back("const_offsets");
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("const_offsets");
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getResultTypes();
|
|
}
|
|
|
|
void LoadMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadMatrixOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadMatrixOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::LoadNdOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
LoadNdOpGenericAdaptorBase::LoadNdOpGenericAdaptorBase(LoadNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> LoadNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> LoadNdOpGenericAdaptorBase::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> LoadNdOpGenericAdaptorBase::getPacked() {
|
|
auto attr = getPackedAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> LoadNdOpGenericAdaptorBase::getTranspose() {
|
|
auto attr = getTransposeAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadNdOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
LoadNdOpAdaptor::LoadNdOpAdaptor(LoadNdOp op) : LoadNdOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult LoadNdOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitError(loc, "'xegpu.load_nd' op requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_packed = getProperties().packed; (void)tblgen_packed;
|
|
auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose;
|
|
|
|
if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_packed && !((::llvm::isa<::mlir::UnitAttr>(tblgen_packed))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'packed' failed to satisfy constraint: unit attribute");
|
|
|
|
if (tblgen_transpose && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_transpose))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'transpose' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.load_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> LoadNdOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange LoadNdOp::getOffsetsMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_offsets;
|
|
auto attr = dict.get("const_offsets");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.packed;
|
|
auto attr = dict.get("packed");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `packed` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.transpose;
|
|
auto attr = dict.get("transpose");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `transpose` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute LoadNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_offsets;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_offsets",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.packed;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("packed",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.transpose;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("transpose",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code LoadNdOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_offsets.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.packed.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.transpose.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> LoadNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_offsets")
|
|
return prop.const_offsets;
|
|
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
|
|
if (name == "packed")
|
|
return prop.packed;
|
|
|
|
if (name == "transpose")
|
|
return prop.transpose;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void LoadNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_offsets") {
|
|
prop.const_offsets = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_offsets)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "packed") {
|
|
prop.packed = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.packed)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "transpose") {
|
|
prop.transpose = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.transpose)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void LoadNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets);
|
|
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
|
|
if (prop.packed) attrs.append("packed", prop.packed);
|
|
|
|
if (prop.transpose) attrs.append("transpose", prop.transpose);
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getPackedAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "packed", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "transpose", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.const_offsets)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.packed)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.transpose)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.const_offsets);
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
|
|
writer.writeOptionalAttribute(prop.packed);
|
|
|
|
writer.writeOptionalAttribute(prop.transpose);
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> LoadNdOp::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> LoadNdOp::getPacked() {
|
|
auto attr = getPackedAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::llvm::ArrayRef<int64_t>> LoadNdOp::getTranspose() {
|
|
auto attr = getTransposeAttr();
|
|
return attr ? ::std::optional<::llvm::ArrayRef<int64_t>>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadNdOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void LoadNdOp::setConstOffsets(::llvm::ArrayRef<int64_t> attrValue) {
|
|
getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue);
|
|
}
|
|
|
|
void LoadNdOp::setPacked(bool attrValue) {
|
|
auto &odsProp = getProperties().packed;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadNdOp::setTranspose(::std::optional<::llvm::ArrayRef<int64_t>> attrValue) {
|
|
auto &odsProp = getProperties().transpose;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void LoadNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value TensorDesc, ArrayRef<OpFoldResult> offsets, UnitAttr packed, DenseI64ArrayAttr transpose, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value TensorDesc, ArrayRef<OpFoldResult> offsets, UnitAttr packed, DenseI64ArrayAttr transpose, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (packed) {
|
|
odsState.getOrAddProperties<Properties>().packed = packed;
|
|
}
|
|
if (transpose) {
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(value);
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (packed) {
|
|
odsState.getOrAddProperties<Properties>().packed = packed;
|
|
}
|
|
if (transpose) {
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (packed) {
|
|
odsState.getOrAddProperties<Properties>().packed = packed;
|
|
}
|
|
if (transpose) {
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
odsState.addTypes(value);
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (packed) {
|
|
odsState.getOrAddProperties<Properties>().packed = packed;
|
|
}
|
|
if (transpose) {
|
|
odsState.getOrAddProperties<Properties>().transpose = transpose;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(packed)>(packed), std::forward<decltype(transpose)>(transpose), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<LoadNdOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void LoadNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<LoadNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_packed = getProperties().packed; (void)tblgen_packed;
|
|
auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_packed, "packed")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_transpose, "transpose")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin()))))))
|
|
return emitOpError("failed to verify that all of {value, TensorDesc} have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult LoadNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc;
|
|
(void)TensorDescOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands;
|
|
::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_offsetsAttr;
|
|
::mlir::Type TensorDescRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1);
|
|
::mlir::Type valueRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1);
|
|
|
|
TensorDescOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(TensorDescRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
result.getOrAddProperties<LoadNdOp::Properties>().const_offsets = const_offsetsAttr;
|
|
}
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(TensorDescRawType))
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
valueRawType = type;
|
|
}
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
result.addTypes(valueTypes);
|
|
if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult LoadNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.packed;
|
|
auto packedAttrName = ::mlir::StringAttr::get(ctx, "packed");
|
|
auto attr = dict.get(packedAttrName);
|
|
usedKeys.insert(packedAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for packed in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `packed` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.transpose;
|
|
auto transposeAttrName = ::mlir::StringAttr::get(ctx, "transpose");
|
|
auto attr = dict.get(transposeAttrName);
|
|
usedKeys.insert(transposeAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for transpose in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `transpose` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void LoadNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc();
|
|
printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
elidedProps.push_back("const_offsets");
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("const_offsets");
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc().getType();
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getValue().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
}
|
|
|
|
void LoadNdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadNdOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadNdOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::NbarrierArriveOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
NbarrierArriveOpGenericAdaptorBase::NbarrierArriveOpGenericAdaptorBase(NbarrierArriveOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
NbarrierArriveOpAdaptor::NbarrierArriveOpAdaptor(NbarrierArriveOp op) : NbarrierArriveOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult NbarrierArriveOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierArriveOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute NbarrierArriveOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code NbarrierArriveOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> NbarrierArriveOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void NbarrierArriveOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void NbarrierArriveOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierArriveOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void NbarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value nbarrier) {
|
|
odsState.addOperands(nbarrier);
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value nbarrier) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(nbarrier)>(nbarrier));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value nbarrier) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(nbarrier)>(nbarrier));
|
|
}
|
|
|
|
void NbarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
odsState.addOperands(nbarrier);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier)>(nbarrier));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier)>(nbarrier));
|
|
}
|
|
|
|
void NbarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void NbarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierArriveOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierArriveOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierArriveOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult NbarrierArriveOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand nbarrierRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrierOperands(&nbarrierRawOperand, 1); ::llvm::SMLoc nbarrierOperandsLoc;
|
|
(void)nbarrierOperandsLoc;
|
|
::mlir::Type nbarrierRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> nbarrierTypes(&nbarrierRawType, 1);
|
|
|
|
nbarrierOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(nbarrierRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(nbarrierRawType))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(nbarrierOperands, nbarrierTypes, nbarrierOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void NbarrierArriveOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getNbarrier();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getNbarrier().getType();
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::NbarrierArriveOp)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::NbarrierWaitOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
NbarrierWaitOpGenericAdaptorBase::NbarrierWaitOpGenericAdaptorBase(NbarrierWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
NbarrierWaitOpAdaptor::NbarrierWaitOpAdaptor(NbarrierWaitOp op) : NbarrierWaitOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult NbarrierWaitOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute NbarrierWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code NbarrierWaitOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> NbarrierWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void NbarrierWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void NbarrierWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void NbarrierWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value nbarrier) {
|
|
odsState.addOperands(nbarrier);
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value nbarrier) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(nbarrier)>(nbarrier));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value nbarrier) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(nbarrier)>(nbarrier));
|
|
}
|
|
|
|
void NbarrierWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
odsState.addOperands(nbarrier);
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier)>(nbarrier));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(nbarrier)>(nbarrier));
|
|
}
|
|
|
|
void NbarrierWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void NbarrierWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<NbarrierWaitOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierWaitOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult NbarrierWaitOp::verifyInvariants() {
|
|
return verifyInvariantsImpl();
|
|
}
|
|
|
|
::mlir::ParseResult NbarrierWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand nbarrierRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrierOperands(&nbarrierRawOperand, 1); ::llvm::SMLoc nbarrierOperandsLoc;
|
|
(void)nbarrierOperandsLoc;
|
|
::mlir::Type nbarrierRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> nbarrierTypes(&nbarrierRawType, 1);
|
|
|
|
nbarrierOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(nbarrierRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(nbarrierRawType))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(nbarrierOperands, nbarrierTypes, nbarrierOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void NbarrierWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getNbarrier();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getNbarrier().getType();
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::NbarrierWaitOp)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::PrefetchNdOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
PrefetchNdOpGenericAdaptorBase::PrefetchNdOpGenericAdaptorBase(PrefetchNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> PrefetchNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> PrefetchNdOpGenericAdaptorBase::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchNdOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
PrefetchNdOpAdaptor::PrefetchNdOpAdaptor(PrefetchNdOp op) : PrefetchNdOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult PrefetchNdOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitError(loc, "'xegpu.prefetch_nd' op requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets))))
|
|
return emitError(loc, "'xegpu.prefetch_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.prefetch_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> PrefetchNdOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 1) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange PrefetchNdOp::getOffsetsMutable() {
|
|
auto range = getODSOperandIndexAndLength(1);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_offsets;
|
|
auto attr = dict.get("const_offsets");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute PrefetchNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_offsets;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_offsets",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code PrefetchNdOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_offsets.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> PrefetchNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_offsets")
|
|
return prop.const_offsets;
|
|
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void PrefetchNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_offsets") {
|
|
prop.const_offsets = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_offsets)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void PrefetchNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets);
|
|
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.const_offsets)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void PrefetchNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.const_offsets);
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> PrefetchNdOp::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchNdOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void PrefetchNdOp::setConstOffsets(::llvm::ArrayRef<int64_t> attrValue) {
|
|
getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue);
|
|
}
|
|
|
|
void PrefetchNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value TensorDesc, ArrayRef<OpFoldResult> offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Value TensorDesc, ArrayRef<OpFoldResult> offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<PrefetchNdOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void PrefetchNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult PrefetchNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc;
|
|
(void)TensorDescOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands;
|
|
::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_offsetsAttr;
|
|
::mlir::Type TensorDescRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1);
|
|
|
|
TensorDescOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(TensorDescRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
result.getOrAddProperties<PrefetchNdOp::Properties>().const_offsets = const_offsetsAttr;
|
|
}
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(TensorDescRawType))
|
|
return ::mlir::failure();
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void PrefetchNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc();
|
|
printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
elidedProps.push_back("const_offsets");
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("const_offsets");
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc().getType();
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::PrefetchNdOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::PrefetchNdOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::PrefetchOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
PrefetchOpGenericAdaptorBase::PrefetchOpGenericAdaptorBase(PrefetchOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<uint64_t> PrefetchOpGenericAdaptorBase::getOffsetAlignByte() {
|
|
auto attr = getOffsetAlignByteAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
PrefetchOpAdaptor::PrefetchOpAdaptor(PrefetchOp op) : PrefetchOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult PrefetchOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_offset_align_byte = getProperties().offset_align_byte; (void)tblgen_offset_align_byte;
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.prefetch' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.prefetch' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.prefetch' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_offset_align_byte && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_offset_align_byte))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_offset_align_byte).getType().isSignlessInteger(64)))))
|
|
return emitError(loc, "'xegpu.prefetch' op attribute 'offset_align_byte' failed to satisfy constraint: 64-bit signless integer attribute");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.prefetch' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.offset_align_byte;
|
|
auto attr = dict.get("offset_align_byte");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `offset_align_byte` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute PrefetchOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.offset_align_byte;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("offset_align_byte",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code PrefetchOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.offset_align_byte.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> PrefetchOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
|
|
if (name == "offset_align_byte")
|
|
return prop.offset_align_byte;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void PrefetchOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "offset_align_byte") {
|
|
prop.offset_align_byte = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.offset_align_byte)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void PrefetchOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
|
|
if (prop.offset_align_byte) attrs.append("offset_align_byte", prop.offset_align_byte);
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getOffsetAlignByteAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "offset_align_byte", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.offset_align_byte)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void PrefetchOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
|
|
writer.writeOptionalAttribute(prop.offset_align_byte);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<uint64_t> PrefetchOp::getOffsetAlignByte() {
|
|
auto attr = getOffsetAlignByteAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void PrefetchOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchOp::setOffsetAlignByte(::std::optional<uint64_t> attrValue) {
|
|
auto &odsProp = getProperties().offset_align_byte;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void PrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (offset_align_byte) {
|
|
odsState.getOrAddProperties<Properties>().offset_align_byte = offset_align_byte;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(offset_align_byte)>(offset_align_byte), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(offset_align_byte)>(offset_align_byte), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(source);
|
|
odsState.addOperands(offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (offset_align_byte) {
|
|
odsState.getOrAddProperties<Properties>().offset_align_byte = offset_align_byte;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(offset_align_byte)>(offset_align_byte), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(offset_align_byte)>(offset_align_byte), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void PrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<PrefetchOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void PrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<PrefetchOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::verifyInvariantsImpl() {
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_offset_align_byte = getProperties().offset_align_byte; (void)tblgen_offset_align_byte;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_offset_align_byte, "offset_align_byte")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult PrefetchOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(offsetsRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(sourceOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult PrefetchOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.offset_align_byte;
|
|
auto offset_align_byteAttrName = ::mlir::StringAttr::get(ctx, "offset_align_byte");
|
|
auto attr = dict.get(offset_align_byteAttrName);
|
|
usedKeys.insert(offset_align_byteAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for offset_align_byte in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `offset_align_byte` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void PrefetchOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getOffsets();
|
|
_odsPrinter << "]";
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::PrefetchOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::PrefetchOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::StoreMatrixOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
StoreMatrixOpGenericAdaptorBase::StoreMatrixOpGenericAdaptorBase(StoreMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> StoreMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> StoreMatrixOpGenericAdaptorBase::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> StoreMatrixOpGenericAdaptorBase::getSubgroupBlockIo() {
|
|
auto attr = getSubgroupBlockIoAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreMatrixOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
StoreMatrixOpAdaptor::StoreMatrixOpAdaptor(StoreMatrixOp op) : StoreMatrixOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult StoreMatrixOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitError(loc, "'xegpu.store_matrix' op requires attribute 'const_offsets'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io;
|
|
|
|
if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets))))
|
|
return emitError(loc, "'xegpu.store_matrix' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_subgroup_block_io && !((::llvm::isa<::mlir::UnitAttr>(tblgen_subgroup_block_io))))
|
|
return emitError(loc, "'xegpu.store_matrix' op attribute 'subgroup_block_io' failed to satisfy constraint: unit attribute");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.store_matrix' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> StoreMatrixOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange StoreMatrixOp::getOffsetsMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_offsets;
|
|
auto attr = dict.get("const_offsets");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.subgroup_block_io;
|
|
auto attr = dict.get("subgroup_block_io");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute StoreMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_offsets;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_offsets",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.subgroup_block_io;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("subgroup_block_io",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code StoreMatrixOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_offsets.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.subgroup_block_io.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> StoreMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_offsets")
|
|
return prop.const_offsets;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
|
|
if (name == "subgroup_block_io")
|
|
return prop.subgroup_block_io;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void StoreMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_offsets") {
|
|
prop.const_offsets = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_offsets)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "subgroup_block_io") {
|
|
prop.subgroup_block_io = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.subgroup_block_io)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void StoreMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
|
|
if (prop.subgroup_block_io) attrs.append("subgroup_block_io", prop.subgroup_block_io);
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getSubgroupBlockIoAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "subgroup_block_io", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.const_offsets)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.subgroup_block_io)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.const_offsets);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
|
|
writer.writeOptionalAttribute(prop.subgroup_block_io);
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> StoreMatrixOp::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<bool> StoreMatrixOp::getSubgroupBlockIo() {
|
|
auto attr = getSubgroupBlockIoAttr();
|
|
return attr ? ::std::optional<bool>(attr != nullptr) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreMatrixOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void StoreMatrixOp::setConstOffsets(::llvm::ArrayRef<int64_t> attrValue) {
|
|
getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue);
|
|
}
|
|
|
|
void StoreMatrixOp::setSubgroupBlockIo(bool attrValue) {
|
|
auto &odsProp = getProperties().subgroup_block_io;
|
|
if (attrValue)
|
|
odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value data, TypedValue<MemDescType> mem_desc, llvm::ArrayRef<OpFoldResult> offsets, DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, Value data, TypedValue<MemDescType> mem_desc, llvm::ArrayRef<OpFoldResult> offsets, DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(data);
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(data);
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(data);
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(data);
|
|
odsState.addOperands(mem_desc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (subgroup_block_io) {
|
|
odsState.getOrAddProperties<Properties>().subgroup_block_io = subgroup_block_io;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(data)>(data), std::forward<decltype(mem_desc)>(mem_desc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(subgroup_block_io)>(subgroup_block_io), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<StoreMatrixOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void StoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreMatrixOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'");
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_subgroup_block_io, "subgroup_block_io")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))))))
|
|
return emitOpError("failed to verify that all of {mem_desc, data} have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult StoreMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand dataRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dataOperands(&dataRawOperand, 1); ::llvm::SMLoc dataOperandsLoc;
|
|
(void)dataOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand mem_descRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mem_descOperands(&mem_descRawOperand, 1); ::llvm::SMLoc mem_descOperandsLoc;
|
|
(void)mem_descOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands;
|
|
::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_offsetsAttr;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
|
|
dataOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(dataRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
mem_descOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(mem_descRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
result.getOrAddProperties<StoreMatrixOp::Properties>().const_offsets = const_offsetsAttr;
|
|
}
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(dataOperands, mem_descOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreMatrixOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.subgroup_block_io;
|
|
auto subgroup_block_ioAttrName = ::mlir::StringAttr::get(ctx, "subgroup_block_io");
|
|
auto attr = dict.get(subgroup_block_ioAttrName);
|
|
usedKeys.insert(subgroup_block_ioAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for subgroup_block_io in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getData();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMemDesc();
|
|
printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
elidedProps.push_back("const_offsets");
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("const_offsets");
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
}
|
|
|
|
void StoreMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreMatrixOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreMatrixOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::StoreNdOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
StoreNdOpGenericAdaptorBase::StoreNdOpGenericAdaptorBase(StoreNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
std::pair<unsigned, unsigned> StoreNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (odsOperandsSize - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> StoreNdOpGenericAdaptorBase::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreNdOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
StoreNdOpAdaptor::StoreNdOpAdaptor(StoreNdOp op) : StoreNdOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult StoreNdOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitError(loc, "'xegpu.store_nd' op requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets))))
|
|
return emitError(loc, "'xegpu.store_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute");
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.store_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.store_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.store_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.store_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> StoreNdOp::getODSOperandIndexAndLength(unsigned index) {
|
|
bool isVariadic[] = {false, false, true};
|
|
int prevVariadicCount = 0;
|
|
for (unsigned i = 0; i < index; ++i)
|
|
if (isVariadic[i]) ++prevVariadicCount;
|
|
|
|
// Calculate how many dynamic values a static variadic operand corresponds to.
|
|
// This assumes all static variadic operands have the same dynamic value count.
|
|
int variadicSize = (getOperation()->getNumOperands() - 2) / 1;
|
|
// `index` passed in as the parameter is the static index which counts each
|
|
// operand (variadic or not) as size 1. So here for each previous static variadic
|
|
// operand, we need to offset by (variadicSize - 1) to get where the dynamic
|
|
// value pack for this static operand starts.
|
|
int start = index + (variadicSize - 1) * prevVariadicCount;
|
|
int size = isVariadic[index] ? variadicSize : 1;
|
|
return {start, size};
|
|
}
|
|
|
|
::mlir::MutableOperandRange StoreNdOp::getOffsetsMutable() {
|
|
auto range = getODSOperandIndexAndLength(2);
|
|
auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second);
|
|
return mutableRange;
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.const_offsets;
|
|
auto attr = dict.get("const_offsets");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute StoreNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.const_offsets;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("const_offsets",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code StoreNdOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.const_offsets.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> StoreNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "const_offsets")
|
|
return prop.const_offsets;
|
|
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void StoreNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "const_offsets") {
|
|
prop.const_offsets = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.const_offsets)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void StoreNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets);
|
|
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readAttribute(prop.const_offsets)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
writer.writeAttribute(prop.const_offsets);
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
}
|
|
|
|
::llvm::ArrayRef<int64_t> StoreNdOp::getConstOffsets() {
|
|
auto attr = getConstOffsetsAttr();
|
|
return attr;
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreNdOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void StoreNdOp::setConstOffsets(::llvm::ArrayRef<int64_t> attrValue) {
|
|
getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue);
|
|
}
|
|
|
|
void StoreNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void StoreNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void StoreNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value TensorDesc, ArrayRef<OpFoldResult> offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value TensorDesc, ArrayRef<OpFoldResult> offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = const_offsets;
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(TensorDesc);
|
|
odsState.addOperands(offsets);
|
|
odsState.getOrAddProperties<Properties>().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets);
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef<int64_t> const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(TensorDesc)>(TensorDesc), std::forward<decltype(offsets)>(offsets), std::forward<decltype(const_offsets)>(const_offsets), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<StoreNdOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void StoreNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() >= 2u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreNdOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::verifyInvariantsImpl() {
|
|
auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets;
|
|
if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'");
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))))))
|
|
return emitOpError("failed to verify that all of {value, TensorDesc} have same element type");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult StoreNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc;
|
|
(void)valueOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc;
|
|
(void)TensorDescOperandsLoc;
|
|
::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands;
|
|
::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::DenseI64ArrayAttr const_offsetsAttr;
|
|
::mlir::Type valueRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1);
|
|
::mlir::Type TensorDescRawType{};
|
|
::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1);
|
|
|
|
valueOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(valueRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
TensorDescOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(TensorDescRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr);
|
|
if (odsResult) return ::mlir::failure();
|
|
result.getOrAddProperties<StoreNdOp::Properties>().const_offsets = const_offsetsAttr;
|
|
}
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
{
|
|
::mlir::VectorType type;
|
|
if (parser.parseCustomTypeWithFallback(type))
|
|
return ::mlir::failure();
|
|
valueRawType = type;
|
|
}
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseType(TensorDescRawType))
|
|
return ::mlir::failure();
|
|
::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType();
|
|
if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getValue();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc();
|
|
printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr());
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
elidedProps.push_back("const_offsets");
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
elidedAttrs.push_back("const_offsets");
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
{
|
|
auto type = getValue().getType();
|
|
if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type))
|
|
_odsPrinter.printStrippedAttrOrType(validType);
|
|
else
|
|
_odsPrinter << type;
|
|
}
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getTensorDesc().getType();
|
|
}
|
|
|
|
void StoreNdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreNdOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreNdOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::StoreScatterOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
StoreScatterOpGenericAdaptorBase::StoreScatterOpGenericAdaptorBase(StoreScatterOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
::std::optional<uint64_t> StoreScatterOpGenericAdaptorBase::getChunkSize() {
|
|
auto attr = getChunkSizeAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreScatterOpGenericAdaptorBase::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
|
|
} // namespace detail
|
|
StoreScatterOpAdaptor::StoreScatterOpAdaptor(StoreScatterOp op) : StoreScatterOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult StoreScatterOpAdaptor::verify(::mlir::Location loc) {
|
|
auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size;
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (tblgen_chunk_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_chunk_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_chunk_size).getType().isSignlessInteger(64)))))
|
|
return emitError(loc, "'xegpu.store' op attribute 'chunk_size' failed to satisfy constraint: 64-bit signless integer attribute");
|
|
|
|
if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint))))
|
|
return emitError(loc, "'xegpu.store' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint))))
|
|
return emitError(loc, "'xegpu.store' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint))))
|
|
return emitError(loc, "'xegpu.store' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators");
|
|
|
|
if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout))))
|
|
return emitError(loc, "'xegpu.store' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.chunk_size;
|
|
auto attr = dict.get("chunk_size");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l1_hint;
|
|
auto attr = dict.get("l1_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l2_hint;
|
|
auto attr = dict.get("l2_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.l3_hint;
|
|
auto attr = dict.get("l3_hint");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
auto &propStorage = prop.layout;
|
|
auto attr = dict.get("layout");
|
|
if (attr) {
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute StoreScatterOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
{
|
|
const auto &propStorage = prop.chunk_size;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("chunk_size",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l1_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l1_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l2_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l2_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.l3_hint;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("l3_hint",
|
|
propStorage));
|
|
}
|
|
|
|
{
|
|
const auto &propStorage = prop.layout;
|
|
if (propStorage)
|
|
attrs.push_back(odsBuilder.getNamedAttr("layout",
|
|
propStorage));
|
|
}
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code StoreScatterOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine(
|
|
llvm::hash_value(prop.chunk_size.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l1_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l2_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.l3_hint.getAsOpaquePointer()),
|
|
llvm::hash_value(prop.layout.getAsOpaquePointer()));
|
|
}
|
|
|
|
std::optional<mlir::Attribute> StoreScatterOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
if (name == "chunk_size")
|
|
return prop.chunk_size;
|
|
|
|
if (name == "l1_hint")
|
|
return prop.l1_hint;
|
|
|
|
if (name == "l2_hint")
|
|
return prop.l2_hint;
|
|
|
|
if (name == "l3_hint")
|
|
return prop.l3_hint;
|
|
|
|
if (name == "layout")
|
|
return prop.layout;
|
|
return std::nullopt;
|
|
}
|
|
|
|
void StoreScatterOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
if (name == "chunk_size") {
|
|
prop.chunk_size = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.chunk_size)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l1_hint") {
|
|
prop.l1_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l1_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l2_hint") {
|
|
prop.l2_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l2_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "l3_hint") {
|
|
prop.l3_hint = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.l3_hint)>>(value);
|
|
return;
|
|
}
|
|
|
|
if (name == "layout") {
|
|
prop.layout = ::llvm::dyn_cast_or_null<std::remove_reference_t<decltype(prop.layout)>>(value);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void StoreScatterOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
if (prop.chunk_size) attrs.append("chunk_size", prop.chunk_size);
|
|
|
|
if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint);
|
|
|
|
if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint);
|
|
|
|
if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint);
|
|
|
|
if (prop.layout) attrs.append("layout", prop.layout);
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getChunkSizeAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "chunk_size", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
{
|
|
::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName));
|
|
if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError)))
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) {
|
|
auto &prop = state.getOrAddProperties<Properties>(); (void)prop;
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.chunk_size)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint)))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(reader.readOptionalAttribute(prop.layout)))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreScatterOp::writeProperties(::mlir::DialectBytecodeWriter &writer) {
|
|
auto &prop = getProperties(); (void)prop;
|
|
|
|
writer.writeOptionalAttribute(prop.chunk_size);
|
|
|
|
writer.writeOptionalAttribute(prop.l1_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l2_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.l3_hint);
|
|
|
|
writer.writeOptionalAttribute(prop.layout);
|
|
}
|
|
|
|
::std::optional<uint64_t> StoreScatterOp::getChunkSize() {
|
|
auto attr = getChunkSizeAttr();
|
|
return attr ? ::std::optional<uint64_t>(attr.getValue().getZExtValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL1Hint() {
|
|
auto attr = getL1HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL2Hint() {
|
|
auto attr = getL2HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL3Hint() {
|
|
auto attr = getL3HintAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt);
|
|
}
|
|
|
|
::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreScatterOp::getLayout() {
|
|
auto attr = getLayoutAttr();
|
|
return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt);
|
|
}
|
|
|
|
void StoreScatterOp::setChunkSize(::std::optional<uint64_t> attrValue) {
|
|
auto &odsProp = getProperties().chunk_size;
|
|
if (attrValue)
|
|
odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void StoreScatterOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l1_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void StoreScatterOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l2_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
void StoreScatterOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) {
|
|
auto &odsProp = getProperties().l3_hint;
|
|
if (attrValue)
|
|
odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue);
|
|
else
|
|
odsProp = nullptr;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value dest, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value dest, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint));
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value dest, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value dest, ArrayRef<OpFoldResult> offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreScatterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(dest);
|
|
odsState.addOperands(offsets);
|
|
odsState.addOperands(mask);
|
|
if (chunk_size) {
|
|
odsState.getOrAddProperties<Properties>().chunk_size = chunk_size;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreScatterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
odsState.addOperands(value);
|
|
odsState.addOperands(dest);
|
|
odsState.addOperands(offsets);
|
|
odsState.addOperands(mask);
|
|
if (chunk_size) {
|
|
odsState.getOrAddProperties<Properties>().chunk_size = chunk_size;
|
|
}
|
|
if (l1_hint) {
|
|
odsState.getOrAddProperties<Properties>().l1_hint = l1_hint;
|
|
}
|
|
if (l2_hint) {
|
|
odsState.getOrAddProperties<Properties>().l2_hint = l2_hint;
|
|
}
|
|
if (l3_hint) {
|
|
odsState.getOrAddProperties<Properties>().l3_hint = l3_hint;
|
|
}
|
|
if (layout) {
|
|
odsState.getOrAddProperties<Properties>().layout = layout;
|
|
}
|
|
assert(resultTypes.size() == 0u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(value)>(value), std::forward<decltype(dest)>(dest), std::forward<decltype(offsets)>(offsets), std::forward<decltype(mask)>(mask), std::forward<decltype(chunk_size)>(chunk_size), std::forward<decltype(l1_hint)>(l1_hint), std::forward<decltype(l2_hint)>(l2_hint), std::forward<decltype(l3_hint)>(l3_hint), std::forward<decltype(layout)>(layout));
|
|
}
|
|
|
|
void StoreScatterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
|
|
if (!attributes.empty()) {
|
|
(void)odsState.getOrAddProperties<StoreScatterOp::Properties>();
|
|
::mlir::PropertyRef properties = odsState.getRawProperties();
|
|
std::optional<::mlir::RegisteredOperationName> info =
|
|
odsState.name.getRegisteredInfo();
|
|
if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties,
|
|
odsState.attributes.getDictionary(odsState.getContext()), nullptr)))
|
|
::llvm::report_fatal_error("Property conversion failed.");
|
|
}
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void StoreScatterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 4u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 0u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<StoreScatterOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::verifyInvariantsImpl() {
|
|
auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size;
|
|
auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint;
|
|
auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint;
|
|
auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint;
|
|
auto tblgen_layout = getProperties().layout; (void)tblgen_layout;
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_chunk_size, "chunk_size")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint")))
|
|
return ::mlir::failure();
|
|
|
|
if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout")))
|
|
return ::mlir::failure();
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup1 = getODSOperands(1);
|
|
|
|
for (auto v : valueGroup1) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup2 = getODSOperands(2);
|
|
|
|
for (auto v : valueGroup2) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
auto valueGroup3 = getODSOperands(3);
|
|
|
|
for (auto v : valueGroup3) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU15(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult StoreScatterOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc;
|
|
(void)valueOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand destRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> destOperands(&destRawOperand, 1); ::llvm::SMLoc destOperandsLoc;
|
|
(void)destOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc;
|
|
(void)offsetsOperandsLoc;
|
|
::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc;
|
|
(void)maskOperandsLoc;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
|
|
valueOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(valueRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
destOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(destRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseLSquare())
|
|
return ::mlir::failure();
|
|
|
|
offsetsOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(offsetsRawOperand))
|
|
return ::mlir::failure();
|
|
if (parser.parseRSquare())
|
|
return ::mlir::failure();
|
|
if (parser.parseComma())
|
|
return ::mlir::failure();
|
|
|
|
maskOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(maskRawOperand))
|
|
return ::mlir::failure();
|
|
if (parseProperties(parser, result))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.resolveOperands(::llvm::concat<const ::mlir::OpAsmParser::UnresolvedOperand>(valueOperands, destOperands, offsetsOperands, maskOperands), allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult StoreScatterOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
// keep track of used keys in the input dictionary to be able to error out
|
|
// if there are some unknown ones.
|
|
::mlir::DenseSet<::mlir::StringAttr> usedKeys;
|
|
::mlir::MLIRContext *ctx = dict.getContext();
|
|
(void)ctx;
|
|
{
|
|
|
|
auto &propStorage = prop.chunk_size;
|
|
auto chunk_sizeAttrName = ::mlir::StringAttr::get(ctx, "chunk_size");
|
|
auto attr = dict.get(chunk_sizeAttrName);
|
|
usedKeys.insert(chunk_sizeAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for chunk_size in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l1_hint;
|
|
auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint");
|
|
auto attr = dict.get(l1_hintAttrName);
|
|
usedKeys.insert(l1_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l1_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l2_hint;
|
|
auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint");
|
|
auto attr = dict.get(l2_hintAttrName);
|
|
usedKeys.insert(l2_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l2_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.l3_hint;
|
|
auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint");
|
|
auto attr = dict.get(l3_hintAttrName);
|
|
usedKeys.insert(l3_hintAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for l3_hint in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
{
|
|
|
|
auto &propStorage = prop.layout;
|
|
auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout");
|
|
auto attr = dict.get(layoutAttrName);
|
|
usedKeys.insert(layoutAttrName);
|
|
if (attr || /*isRequired=*/false) {
|
|
if (!attr) {
|
|
emitError() << "expected key entry for layout in DictionaryAttr to set "
|
|
"Properties.";
|
|
return ::mlir::failure();
|
|
}
|
|
auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
|
|
if (convertedAttr) {
|
|
propStorage = convertedAttr;
|
|
} else {
|
|
emitError() << "Invalid attribute `layout` in property conversion: " << attr;
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
}
|
|
|
|
for (::mlir::NamedAttribute attr : dict) {
|
|
if (!usedKeys.contains(attr.getName()))
|
|
return emitError() << "unknown key '" << attr.getName() <<
|
|
"' when parsing properties dictionary";
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void StoreScatterOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getValue();
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getDest();
|
|
_odsPrinter << "[";
|
|
_odsPrinter << getOffsets();
|
|
_odsPrinter << "]";
|
|
_odsPrinter << ",";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getMask();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;
|
|
_odsPrinter << " ";
|
|
printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps);
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
}
|
|
|
|
void StoreScatterOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get());
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreScatterOp)
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreScatterOpGenericAdaptorBase::Properties)
|
|
|
|
namespace mlir::xegpu {
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// ::mlir::xegpu::TruncfOp definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace detail {
|
|
|
|
TruncfOpGenericAdaptorBase::TruncfOpGenericAdaptorBase(TruncfOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {}
|
|
|
|
|
|
} // namespace detail
|
|
TruncfOpAdaptor::TruncfOpAdaptor(TruncfOp op) : TruncfOpGenericAdaptor(op->getOperands(), op) {}
|
|
|
|
::llvm::LogicalResult TruncfOpAdaptor::verify(::mlir::Location loc) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TruncfOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
|
|
if (!dict) {
|
|
emitError() << "expected DictionaryAttr to set properties";
|
|
return ::mlir::failure();
|
|
}
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::mlir::Attribute TruncfOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) {
|
|
::mlir::SmallVector<::mlir::NamedAttribute> attrs;
|
|
::mlir::Builder odsBuilder{ctx};
|
|
|
|
if (!attrs.empty())
|
|
return odsBuilder.getDictionaryAttr(attrs);
|
|
return {};
|
|
}
|
|
|
|
llvm::hash_code TruncfOp::computePropertiesHash(const Properties &prop) {
|
|
using llvm::hash_value;
|
|
return llvm::hash_combine();
|
|
}
|
|
|
|
std::optional<mlir::Attribute> TruncfOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
void TruncfOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) {
|
|
}
|
|
|
|
void TruncfOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) {
|
|
}
|
|
|
|
::llvm::LogicalResult TruncfOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TruncfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value source) {
|
|
odsState.addOperands(source);
|
|
odsState.addTypes(result);
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value source) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(result)>(result), std::forward<decltype(source)>(source));
|
|
auto __res__ = ::llvm::dyn_cast<TruncfOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value source) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(result)>(result), std::forward<decltype(source)>(source));
|
|
}
|
|
|
|
void TruncfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
odsState.addOperands(source);
|
|
assert(resultTypes.size() == 1u && "mismatched number of results");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source));
|
|
auto __res__ = ::llvm::dyn_cast<TruncfOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(source)>(source));
|
|
}
|
|
|
|
void TruncfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.addAttributes(attributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
auto __res__ = ::llvm::dyn_cast<TruncfOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(attributes)>(attributes));
|
|
}
|
|
|
|
void TruncfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
assert(operands.size() == 1u && "mismatched number of parameters");
|
|
odsState.addOperands(operands);
|
|
odsState.useProperties(const_cast<Properties&>(properties));
|
|
odsState.addAttributes(discardableAttributes);
|
|
assert(resultTypes.size() == 1u && "mismatched number of return types");
|
|
odsState.addTypes(resultTypes);
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
::mlir::OperationState __state__(location, getOperationName());
|
|
build(builder, __state__, std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
auto __res__ = ::llvm::dyn_cast<TruncfOp>(builder.create(__state__));
|
|
assert(__res__ && "builder didn't return the right type");
|
|
return __res__;
|
|
}
|
|
|
|
TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) {
|
|
return create(builder, builder.getLoc(), std::forward<decltype(resultTypes)>(resultTypes), std::forward<decltype(operands)>(operands), std::forward<decltype(properties)>(properties), std::forward<decltype(discardableAttributes)>(discardableAttributes));
|
|
}
|
|
|
|
::llvm::LogicalResult TruncfOp::verifyInvariantsImpl() {
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSOperands(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU17(*this, v.getType(), "operand", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
{
|
|
unsigned index = 0; (void)index;
|
|
auto valueGroup0 = getODSResults(0);
|
|
|
|
for (auto v : valueGroup0) {
|
|
if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU17(*this, v.getType(), "result", index++)))
|
|
return ::mlir::failure();
|
|
}
|
|
}
|
|
if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getRank()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getRank()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getRank()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getRank()))))
|
|
return emitOpError("failed to verify that all of {source, result} have same rank");
|
|
if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()))))
|
|
return emitOpError("failed to verify that all of {source, result} have same shape");
|
|
return ::mlir::success();
|
|
}
|
|
|
|
::llvm::LogicalResult TruncfOp::verifyInvariants() {
|
|
if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify()))
|
|
return ::mlir::success();
|
|
return ::mlir::failure();
|
|
}
|
|
|
|
::mlir::ParseResult TruncfOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) {
|
|
::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{};
|
|
::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc;
|
|
(void)sourceOperandsLoc;
|
|
::llvm::SmallVector<::mlir::Type, 1> allOperandTypes;
|
|
::llvm::SmallVector<::mlir::Type, 1> allResultTypes;
|
|
|
|
sourceOperandsLoc = parser.getCurrentLocation();
|
|
if (parser.parseOperand(sourceRawOperand))
|
|
return ::mlir::failure();
|
|
{
|
|
auto loc = parser.getCurrentLocation();(void)loc;
|
|
if (parser.parseOptionalAttrDict(result.attributes))
|
|
return ::mlir::failure();
|
|
if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() {
|
|
return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op ";
|
|
})))
|
|
return ::mlir::failure();
|
|
}
|
|
if (parser.parseColon())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allOperandTypes))
|
|
return ::mlir::failure();
|
|
if (parser.parseArrow())
|
|
return ::mlir::failure();
|
|
|
|
if (parser.parseTypeList(allResultTypes))
|
|
return ::mlir::failure();
|
|
result.addTypes(allResultTypes);
|
|
if (parser.resolveOperands(sourceOperands, allOperandTypes, parser.getNameLoc(), result.operands))
|
|
return ::mlir::failure();
|
|
return ::mlir::success();
|
|
}
|
|
|
|
void TruncfOp::print(::mlir::OpAsmPrinter &_odsPrinter) {
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getSource();
|
|
::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs;
|
|
_odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
|
|
_odsPrinter << ' ' << ":";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getOperandTypes();
|
|
_odsPrinter << ' ' << "->";
|
|
_odsPrinter << ' ';
|
|
_odsPrinter << getOperation()->getResultTypes();
|
|
}
|
|
|
|
void TruncfOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) {
|
|
}
|
|
|
|
|
|
} // namespace mlir::xegpu
|
|
MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::TruncfOp)
|
|
|
|
|
|
#endif // GET_OP_CLASSES
|
|
|