/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ |* *| |* Op Definitions *| |* *| |* Automatically generated file, do not edit! *| |* From: XeGPU.td *| |* *| \*===----------------------------------------------------------------------===*/ #ifdef GET_OP_LIST #undef GET_OP_LIST ::mlir::xegpu::AllocNbarrierOp, ::mlir::xegpu::AtomicRMWOp, ::mlir::xegpu::ConvertLayoutOp, ::mlir::xegpu::CreateMemDescOp, ::mlir::xegpu::CreateNdDescOp, ::mlir::xegpu::DpasMxOp, ::mlir::xegpu::DpasOp, ::mlir::xegpu::FenceOp, ::mlir::xegpu::InitNbarrierOp, ::mlir::xegpu::LoadGatherOp, ::mlir::xegpu::LoadMatrixOp, ::mlir::xegpu::LoadNdOp, ::mlir::xegpu::NbarrierArriveOp, ::mlir::xegpu::NbarrierWaitOp, ::mlir::xegpu::PrefetchNdOp, ::mlir::xegpu::PrefetchOp, ::mlir::xegpu::StoreMatrixOp, ::mlir::xegpu::StoreNdOp, ::mlir::xegpu::StoreScatterOp, ::mlir::xegpu::TruncfOp #endif // GET_OP_LIST #ifdef GET_OP_CLASSES #undef GET_OP_CLASSES //===----------------------------------------------------------------------===// // Local Utility Method Definitions //===----------------------------------------------------------------------===// static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU1( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::xegpu::TensorDescType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be TensorDesc describing regions of interested data., but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU2( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(1)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of 1-bit signless integer values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU3( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 8)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4/5/6/7/8, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU4( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return ((((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType))))) || ((::llvm::isa<::mlir::IndexType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 8))))) || ((((type.isSignlessInteger(1))) || ((type.isSignlessInteger(4))) || ((type.isSignlessInteger(8))) || ((type.isSignlessInteger(16))) || ((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignedInteger(1))) || ((type.isSignedInteger(8))) || ((type.isSignedInteger(16))) || ((type.isSignedInteger(32))) || ((type.isSignedInteger(64))) || ((type.isUnsignedInteger(1))) || ((type.isUnsignedInteger(8))) || ((type.isUnsignedInteger(16))) || ((type.isUnsignedInteger(32))) || ((type.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(type))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(type))) || ((::llvm::isa<::mlir::Float8E5M2Type>(type))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || ((type.isF16())) || ((type.isF32())) || ((type.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(type))) || ((::llvm::isa<::mlir::FloatTF32Type>(type))))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type or index values of ranks 1/2/3/4/5/6/7/8 or 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU5( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() == 1)))) && ((::llvm::cast<::mlir::ShapedType>(type).hasStaticShape())) && ((XeGPUDialect::isSharedMemory(llvm::cast(type))))) || (((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() == 2)))) && ((::llvm::cast<::mlir::ShapedType>(type).hasStaticShape())) && ((XeGPUDialect::isSharedMemory(llvm::cast(type))))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be reside in share memory and statically 1d shaped memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or reside in share memory and statically 2d shaped memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values , but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU6( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::xegpu::MemDescType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be MemDesc describing the data in SLM, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU7( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() >= 1)))) || (((type.isUnsignedInteger(64))) || ((type.isUnsignedInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignlessInteger(32)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be non-0-ranked.memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or 64-bit unsigned integer or 32-bit unsigned integer or 64-bit signless integer or 32-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU8( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU9( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 4)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU10( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || (((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::Float8E8M0FNUType>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 4))))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be f8E8M0FNU type or vector of f8E8M0FNU type values of ranks 1/2/3/4, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU11( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((type.isSignlessInteger(8)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 8-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU12( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::xegpu::NbarrierType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be !xegpu.nbarrier a custom XeGPU type representing a barrier., but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU13( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() == 1)))) || (((type.isUnsignedInteger(64))) || ((type.isUnsignedInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignlessInteger(32)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 1D memref of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values or 64-bit unsigned integer or 32-bit unsigned integer or 64-bit signless integer or 32-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU14( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::IndexType>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((::llvm::isa<::mlir::IndexType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of index values or index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU15( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(1)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((type.isSignlessInteger(1))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of 1-bit signless integer values or 1-bit signless integer, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU16( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return (((elementType.isSignlessInteger(1))) || ((elementType.isSignlessInteger(4))) || ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignedInteger(1))) || ((elementType.isSignedInteger(8))) || ((elementType.isSignedInteger(16))) || ((elementType.isSignedInteger(32))) || ((elementType.isSignedInteger(64))) || ((elementType.isUnsignedInteger(1))) || ((elementType.isUnsignedInteger(8))) || ((elementType.isUnsignedInteger(16))) || ((elementType.isUnsignedInteger(32))) || ((elementType.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType)))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && (((::llvm::cast<::mlir::VectorType>(type).getRank() == 1)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 3)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 5)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 6)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 7)) || ((::llvm::cast<::mlir::VectorType>(type).getRank() == 8))))) || ((((type.isSignlessInteger(1))) || ((type.isSignlessInteger(4))) || ((type.isSignlessInteger(8))) || ((type.isSignlessInteger(16))) || ((type.isSignlessInteger(32))) || ((type.isSignlessInteger(64))) || ((type.isSignedInteger(1))) || ((type.isSignedInteger(8))) || ((type.isSignedInteger(16))) || ((type.isSignedInteger(32))) || ((type.isSignedInteger(64))) || ((type.isUnsignedInteger(1))) || ((type.isUnsignedInteger(8))) || ((type.isUnsignedInteger(16))) || ((type.isUnsignedInteger(32))) || ((type.isUnsignedInteger(64)))) || (((::llvm::isa<::mlir::Float4E2M1FNType>(type))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(type))) || ((::llvm::isa<::mlir::Float8E5M2Type>(type))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(type))) || ((type.isF16())) || ((type.isF32())) || ((type.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(type))) || ((::llvm::isa<::mlir::FloatTF32Type>(type))))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values of ranks 1/2/3/4/5/6/7/8 or 1-bit signless integer or 4-bit signless integer or 8-bit signless integer or 16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 1-bit signed integer or 8-bit signed integer or 16-bit signed integer or 32-bit signed integer or 64-bit signed integer or 1-bit unsigned integer or 8-bit unsigned integer or 16-bit unsigned integer or 32-bit unsigned integer or 64-bit unsigned integer or f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_XeGPU17( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::FixedVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return ((::llvm::isa<::mlir::Float4E2M1FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E4M3FNType>(elementType))) || ((::llvm::isa<::mlir::Float8E5M2Type>(elementType))) || ((::llvm::isa<::mlir::Float8E8M0FNUType>(elementType))) || ((elementType.isF16())) || ((elementType.isF32())) || ((elementType.isF64())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((::llvm::isa<::mlir::FloatTF32Type>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be fixed-length vector of f4E2M1FN type or f8E4M3FN type or f8E5M2 type or f8E8M0FNU type or 16-bit float or 32-bit float or 64-bit float or bfloat16 type or tf32 type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU1( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !(((::llvm::isa<::mlir::IntegerAttr>(attr))) && ((::llvm::cast<::mlir::IntegerAttr>(attr).getType().isSignlessInteger(64))))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: 64-bit signless integer attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU1( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU1(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU2( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::arith::AtomicRMWKindAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: allowed 64-bit signless integer cases: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU2( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU2(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU3( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: DistributeLayoutAttr instance"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU3( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU3(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU4( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: i64 dense array attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU4( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU4(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU5( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::xegpu::MemorySpaceAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Describe the location of data described by a `TensorDesc`:\n Global device memory (`Global`) or Shared local memory (`SLM`)."; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU5( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU5(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU6( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::xegpu::FenceScopeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Describes the scope of fence.\n \"workgroup\" means that the scope is within each work group.\n \"gpu\" means the scope is across work groups within the gpu."; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU6( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU6(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU7( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU7( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU7(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU8( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::UnitAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: unit attribute"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_XeGPU8( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_XeGPU8(attr, attrName, [op]() { return op->emitOpError(); }); } namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::AllocNbarrierOp definitions //===----------------------------------------------------------------------===// namespace detail { AllocNbarrierOpGenericAdaptorBase::AllocNbarrierOpGenericAdaptorBase(AllocNbarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} uint64_t AllocNbarrierOpGenericAdaptorBase::getNbarrierNum() { auto attr = getNbarrierNumAttr(); return attr.getValue().getZExtValue(); } } // namespace detail AllocNbarrierOpAdaptor::AllocNbarrierOpAdaptor(AllocNbarrierOp op) : AllocNbarrierOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult AllocNbarrierOpAdaptor::verify(::mlir::Location loc) { auto tblgen_nbarrier_num = getProperties().nbarrier_num; (void)tblgen_nbarrier_num; if (!tblgen_nbarrier_num) return emitError(loc, "'xegpu.alloc_nbarrier' op requires attribute 'nbarrier_num'"); if (tblgen_nbarrier_num && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_nbarrier_num))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_nbarrier_num).getType().isSignlessInteger(64))))) return emitError(loc, "'xegpu.alloc_nbarrier' op attribute 'nbarrier_num' failed to satisfy constraint: 64-bit signless integer attribute"); return ::mlir::success(); } ::llvm::LogicalResult AllocNbarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.nbarrier_num; auto attr = dict.get("nbarrier_num"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `nbarrier_num` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute AllocNbarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.nbarrier_num; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("nbarrier_num", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code AllocNbarrierOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.nbarrier_num.getAsOpaquePointer())); } std::optional AllocNbarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "nbarrier_num") return prop.nbarrier_num; return std::nullopt; } void AllocNbarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "nbarrier_num") { prop.nbarrier_num = ::llvm::dyn_cast_or_null>(value); return; } } void AllocNbarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.nbarrier_num) attrs.append("nbarrier_num", prop.nbarrier_num); } ::llvm::LogicalResult AllocNbarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getNbarrierNumAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "nbarrier_num", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult AllocNbarrierOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.nbarrier_num))) return ::mlir::failure(); return ::mlir::success(); } void AllocNbarrierOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.nbarrier_num); } uint64_t AllocNbarrierOp::getNbarrierNum() { auto attr = getNbarrierNumAttr(); return attr.getValue().getZExtValue(); } void AllocNbarrierOp::setNbarrierNum(uint64_t attrValue) { getProperties().nbarrier_num = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), attrValue); } void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::IntegerAttr nbarrier_num) { odsState.getOrAddProperties().nbarrier_num = nbarrier_num; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::IntegerAttr nbarrier_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(nbarrier_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::IntegerAttr nbarrier_num) { return create(builder, builder.getLoc(), std::forward(nbarrier_num)); } void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) { odsState.getOrAddProperties().nbarrier_num = nbarrier_num; assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(nbarrier_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::IntegerAttr nbarrier_num) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(nbarrier_num)); } void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, uint64_t nbarrier_num) { odsState.getOrAddProperties().nbarrier_num = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), nbarrier_num); } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, uint64_t nbarrier_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(nbarrier_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, uint64_t nbarrier_num) { return create(builder, builder.getLoc(), std::forward(nbarrier_num)); } void AllocNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) { odsState.getOrAddProperties().nbarrier_num = odsBuilder.getIntegerAttr(odsBuilder.getIntegerType(64), nbarrier_num); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(nbarrier_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, uint64_t nbarrier_num) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(nbarrier_num)); } void AllocNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void AllocNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } AllocNbarrierOp AllocNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AllocNbarrierOp AllocNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult AllocNbarrierOp::verifyInvariantsImpl() { auto tblgen_nbarrier_num = getProperties().nbarrier_num; (void)tblgen_nbarrier_num; if (!tblgen_nbarrier_num) return emitOpError("requires attribute 'nbarrier_num'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_nbarrier_num, "nbarrier_num"))) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult AllocNbarrierOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult AllocNbarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::IntegerAttr nbarrier_numAttr; if (parser.parseCustomAttributeWithFallback(nbarrier_numAttr, parser.getBuilder().getIntegerType(64))) { return ::mlir::failure(); } if (nbarrier_numAttr) result.getOrAddProperties().nbarrier_num = nbarrier_numAttr; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } return ::mlir::success(); } void AllocNbarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printAttributeWithoutType(getNbarrierNumAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("nbarrier_num"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::AllocNbarrierOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::AllocNbarrierOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::AtomicRMWOp definitions //===----------------------------------------------------------------------===// namespace detail { AtomicRMWOpGenericAdaptorBase::AtomicRMWOpGenericAdaptorBase(AtomicRMWOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::arith::AtomicRMWKind AtomicRMWOpGenericAdaptorBase::getKind() { auto attr = getKindAttr(); return attr.getValue(); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> AtomicRMWOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail AtomicRMWOpAdaptor::AtomicRMWOpAdaptor(AtomicRMWOp op) : AtomicRMWOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult AtomicRMWOpAdaptor::verify(::mlir::Location loc) { auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (!tblgen_kind) return emitError(loc, "'xegpu.atomic_rmw' op requires attribute 'kind'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_kind && !((::llvm::isa<::mlir::arith::AtomicRMWKindAttr>(tblgen_kind)))) return emitError(loc, "'xegpu.atomic_rmw' op attribute 'kind' failed to satisfy constraint: allowed 64-bit signless integer cases: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.atomic_rmw' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } ::llvm::LogicalResult AtomicRMWOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.kind; auto attr = dict.get("kind"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kind` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute AtomicRMWOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.kind; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("kind", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code AtomicRMWOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.kind.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional AtomicRMWOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "kind") return prop.kind; if (name == "layout") return prop.layout; return std::nullopt; } void AtomicRMWOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "kind") { prop.kind = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void AtomicRMWOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.kind) attrs.append("kind", prop.kind); if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult AtomicRMWOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getKindAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU2(attr, "kind", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult AtomicRMWOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.kind))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void AtomicRMWOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.kind); writer.writeOptionalAttribute(prop.layout); } ::mlir::arith::AtomicRMWKind AtomicRMWOp::getKind() { auto attr = getKindAttr(); return attr.getValue(); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> AtomicRMWOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void AtomicRMWOp::setKind(::mlir::arith::AtomicRMWKind attrValue) { getProperties().kind = ::mlir::arith::AtomicRMWKindAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(tensorDesc); odsState.addOperands(mask); odsState.addOperands(value); odsState.getOrAddProperties().kind = kind; if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); } void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(tensorDesc); odsState.addOperands(mask); odsState.addOperands(value); odsState.getOrAddProperties().kind = kind; if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKindAttr kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); } void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(tensorDesc); odsState.addOperands(mask); odsState.addOperands(value); odsState.getOrAddProperties().kind = ::mlir::arith::AtomicRMWKindAttr::get(odsBuilder.getContext(), kind); if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); } void AtomicRMWOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(tensorDesc); odsState.addOperands(mask); odsState.addOperands(value); odsState.getOrAddProperties().kind = ::mlir::arith::AtomicRMWKindAttr::get(odsBuilder.getContext(), kind); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arith::AtomicRMWKind kind, ::mlir::Value tensorDesc, ::mlir::Value mask, ::mlir::Value value, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(kind), std::forward(tensorDesc), std::forward(mask), std::forward(value), std::forward(layout)); } void AtomicRMWOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void AtomicRMWOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } AtomicRMWOp AtomicRMWOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } AtomicRMWOp AtomicRMWOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult AtomicRMWOp::verifyInvariantsImpl() { auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (!tblgen_kind) return emitOpError("requires attribute 'kind'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU2(*this, tblgen_kind, "kind"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(2).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin()))) && (getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {tensorDesc, value, result} have same element type"); if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(2).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(2).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape())))) return emitOpError("failed to verify that all of {tensorDesc, value, result} have same shape"); return ::mlir::success(); } ::llvm::LogicalResult AtomicRMWOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult AtomicRMWOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::arith::AtomicRMWKindAttr kindAttr; ::mlir::OpAsmParser::UnresolvedOperand tensorDescRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tensorDescOperands(&tensorDescRawOperand, 1); ::llvm::SMLoc tensorDescOperandsLoc; (void)tensorDescOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::Type tensorDescRawType{}; ::llvm::ArrayRef<::mlir::Type> tensorDescTypes(&tensorDescRawType, 1); ::mlir::Type maskRawType{}; ::llvm::ArrayRef<::mlir::Type> maskTypes(&maskRawType, 1); ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); { ::llvm::StringRef attrStr; ::mlir::NamedAttrList attrStorage; auto loc = parser.getCurrentLocation(); if (parser.parseOptionalKeyword(&attrStr, {"addf","addi","andi","assign","maximumf","maxnumf","maxs","maxu","minimumf","minnumf","mins","minu","mulf","muli","ori","xori"})) { ::mlir::StringAttr attrVal; ::mlir::OptionalParseResult parseResult = parser.parseOptionalAttribute(attrVal, parser.getBuilder().getNoneType(), "kind", attrStorage); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); attrStr = attrVal.getValue(); } else { return parser.emitError(loc, "expected string or keyword containing one of the following enum values for attribute 'kind' [addf, addi, andi, assign, maximumf, maxnumf, maxs, maxu, minimumf, minnumf, mins, minu, mulf, muli, ori, xori]"); } } if (!attrStr.empty()) { auto attrOptional = ::mlir::arith::symbolizeAtomicRMWKind(attrStr); if (!attrOptional) return parser.emitError(loc, "invalid ") << "kind attribute specification: \"" << attrStr << '"';; kindAttr = ::mlir::arith::AtomicRMWKindAttr::get(parser.getBuilder().getContext(), *attrOptional); result.getOrAddProperties().kind = kindAttr; } } tensorDescOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tensorDescRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); maskOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(maskRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseType(tensorDescRawType)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); maskRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); if (parser.resolveOperands(tensorDescOperands, tensorDescTypes, tensorDescOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(maskOperands, maskTypes, maskOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void AtomicRMWOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; { auto caseValue = getKind(); auto caseValueStr = stringifyAtomicRMWKind(caseValue); _odsPrinter << caseValueStr; } _odsPrinter << ' '; _odsPrinter << getTensorDesc(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMask(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getValue(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("kind"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getTensorDesc().getType(); _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMask().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void AtomicRMWOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::AtomicRMWOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::AtomicRMWOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::ConvertLayoutOp definitions //===----------------------------------------------------------------------===// namespace detail { ConvertLayoutOpGenericAdaptorBase::ConvertLayoutOpGenericAdaptorBase(ConvertLayoutOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOpGenericAdaptorBase::getInputLayout() { auto attr = getInputLayoutAttr(); return attr; } ::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOpGenericAdaptorBase::getTargetLayout() { auto attr = getTargetLayoutAttr(); return attr; } } // namespace detail ConvertLayoutOpAdaptor::ConvertLayoutOpAdaptor(ConvertLayoutOp op) : ConvertLayoutOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ConvertLayoutOpAdaptor::verify(::mlir::Location loc) { auto tblgen_input_layout = getProperties().input_layout; (void)tblgen_input_layout; if (!tblgen_input_layout) return emitError(loc, "'xegpu.convert_layout' op requires attribute 'input_layout'"); auto tblgen_target_layout = getProperties().target_layout; (void)tblgen_target_layout; if (!tblgen_target_layout) return emitError(loc, "'xegpu.convert_layout' op requires attribute 'target_layout'"); if (tblgen_input_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_input_layout)))) return emitError(loc, "'xegpu.convert_layout' op attribute 'input_layout' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_target_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_target_layout)))) return emitError(loc, "'xegpu.convert_layout' op attribute 'target_layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } ::llvm::LogicalResult ConvertLayoutOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.input_layout; auto attr = dict.get("input_layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `input_layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.target_layout; auto attr = dict.get("target_layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `target_layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ConvertLayoutOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.input_layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("input_layout", propStorage)); } { const auto &propStorage = prop.target_layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("target_layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ConvertLayoutOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.input_layout.getAsOpaquePointer()), llvm::hash_value(prop.target_layout.getAsOpaquePointer())); } std::optional ConvertLayoutOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "input_layout") return prop.input_layout; if (name == "target_layout") return prop.target_layout; return std::nullopt; } void ConvertLayoutOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "input_layout") { prop.input_layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "target_layout") { prop.target_layout = ::llvm::dyn_cast_or_null>(value); return; } } void ConvertLayoutOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.input_layout) attrs.append("input_layout", prop.input_layout); if (prop.target_layout) attrs.append("target_layout", prop.target_layout); } ::llvm::LogicalResult ConvertLayoutOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getInputLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "input_layout", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTargetLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "target_layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ConvertLayoutOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.input_layout))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.target_layout))) return ::mlir::failure(); return ::mlir::success(); } void ConvertLayoutOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.input_layout); writer.writeAttribute(prop.target_layout); } ::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOp::getInputLayout() { auto attr = getInputLayoutAttr(); return attr; } ::mlir::xegpu::DistributeLayoutAttr ConvertLayoutOp::getTargetLayout() { auto attr = getTargetLayoutAttr(); return attr; } void ConvertLayoutOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { odsState.addOperands(source); odsState.getOrAddProperties().input_layout = input_layout; odsState.getOrAddProperties().target_layout = target_layout; odsState.addTypes(result); } ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(source), std::forward(input_layout), std::forward(target_layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(source), std::forward(input_layout), std::forward(target_layout)); } void ConvertLayoutOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { odsState.addOperands(source); odsState.getOrAddProperties().input_layout = input_layout; odsState.getOrAddProperties().target_layout = target_layout; assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source), std::forward(input_layout), std::forward(target_layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::xegpu::DistributeLayoutAttr input_layout, ::mlir::xegpu::DistributeLayoutAttr target_layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source), std::forward(input_layout), std::forward(target_layout)); } void ConvertLayoutOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ConvertLayoutOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ConvertLayoutOp ConvertLayoutOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ConvertLayoutOp ConvertLayoutOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ConvertLayoutOp::verifyInvariantsImpl() { auto tblgen_input_layout = getProperties().input_layout; (void)tblgen_input_layout; if (!tblgen_input_layout) return emitOpError("requires attribute 'input_layout'"); auto tblgen_target_layout = getProperties().target_layout; (void)tblgen_target_layout; if (!tblgen_target_layout) return emitOpError("requires attribute 'target_layout'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_input_layout, "input_layout"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_target_layout, "target_layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU4(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {source, result} have same type"); return ::mlir::success(); } ::llvm::LogicalResult ConvertLayoutOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult ConvertLayoutOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::mlir::Type sourceRawType{}; ::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1); sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); sourceRawType = type; } result.addTypes(sourceTypes[0]); if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult ConvertLayoutOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.input_layout; auto input_layoutAttrName = ::mlir::StringAttr::get(ctx, "input_layout"); auto attr = dict.get(input_layoutAttrName); usedKeys.insert(input_layoutAttrName); if (attr || /*isRequired=*/true) { if (!attr) { emitError() << "expected key entry for input_layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `input_layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.target_layout; auto target_layoutAttrName = ::mlir::StringAttr::get(ctx, "target_layout"); auto attr = dict.get(target_layoutAttrName); usedKeys.insert(target_layoutAttrName); if (attr || /*isRequired=*/true) { if (!attr) { emitError() << "expected key entry for target_layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `target_layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void ConvertLayoutOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSource().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void ConvertLayoutOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::ConvertLayoutOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::ConvertLayoutOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::CreateMemDescOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateMemDescOpGenericAdaptorBase::CreateMemDescOpGenericAdaptorBase(CreateMemDescOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail CreateMemDescOpAdaptor::CreateMemDescOpAdaptor(CreateMemDescOp op) : CreateMemDescOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateMemDescOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult CreateMemDescOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateMemDescOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateMemDescOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CreateMemDescOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CreateMemDescOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CreateMemDescOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CreateMemDescOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CreateMemDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type mem_desc, ::mlir::Value source) { odsState.addOperands(source); odsState.addTypes(mem_desc); } CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type mem_desc, ::mlir::Value source) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(mem_desc), std::forward(source)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type mem_desc, ::mlir::Value source) { return create(builder, builder.getLoc(), std::forward(mem_desc), std::forward(source)); } void CreateMemDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source) { odsState.addOperands(source); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source)); } void CreateMemDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateMemDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateMemDescOp CreateMemDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateMemDescOp CreateMemDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateMemDescOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((llvm::cast((*this->getODSOperands(0).begin()).getType()).getNumElements()*llvm::cast((*this->getODSOperands(0).begin()).getType()).getElementTypeBitWidth()) == (llvm::cast((*this->getODSResults(0).begin()).getType()).getNumElements()*llvm::cast((*this->getODSResults(0).begin()).getType()).getElementTypeBitWidth()) && (llvm::cast((*this->getODSResults(0).begin()).getType()).getNumElements()*llvm::cast((*this->getODSResults(0).begin()).getType()).getElementTypeBitWidth()) == (llvm::cast((*this->getODSOperands(0).begin()).getType()).getNumElements()*llvm::cast((*this->getODSOperands(0).begin()).getType()).getElementTypeBitWidth())))) return emitOpError("failed to verify that all of {source, mem_desc} have same size in bits"); return ::mlir::success(); } ::llvm::LogicalResult CreateMemDescOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult CreateMemDescOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::mlir::Type sourceRawType{}; ::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1); ::mlir::Type mem_descRawType{}; ::llvm::ArrayRef<::mlir::Type> mem_descTypes(&mem_descRawType, 1); sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); sourceRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); if (parser.parseType(mem_descRawType)) return ::mlir::failure(); result.addTypes(mem_descTypes); if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult CreateMemDescOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void CreateMemDescOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ":"; _odsPrinter << ' '; { auto type = getSource().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; _odsPrinter << getMemDesc().getType(); } void CreateMemDescOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::CreateMemDescOp) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::CreateNdDescOp definitions //===----------------------------------------------------------------------===// namespace detail { CreateNdDescOpGenericAdaptorBase::CreateNdDescOpGenericAdaptorBase(CreateNdDescOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair CreateNdDescOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::std::optional<::llvm::ArrayRef> CreateNdDescOpGenericAdaptorBase::getConstShape() { auto attr = getConstShapeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> CreateNdDescOpGenericAdaptorBase::getConstStrides() { auto attr = getConstStridesAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } } // namespace detail CreateNdDescOpAdaptor::CreateNdDescOpAdaptor(CreateNdDescOp op) : CreateNdDescOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CreateNdDescOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_shape = getProperties().const_shape; (void)tblgen_const_shape; auto tblgen_const_strides = getProperties().const_strides; (void)tblgen_const_strides; if (tblgen_const_shape && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_shape)))) return emitError(loc, "'xegpu.create_nd_tdesc' op attribute 'const_shape' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_const_strides && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_strides)))) return emitError(loc, "'xegpu.create_nd_tdesc' op attribute 'const_strides' failed to satisfy constraint: i64 dense array attribute"); return ::mlir::success(); } std::pair CreateNdDescOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange CreateNdDescOp::getShapeMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange CreateNdDescOp::getStridesMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult CreateNdDescOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_shape; auto attr = dict.get("const_shape"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_shape` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.const_strides; auto attr = dict.get("const_strides"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_strides` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CreateNdDescOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_shape; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_shape", propStorage)); } { const auto &propStorage = prop.const_strides; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_strides", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CreateNdDescOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_shape.getAsOpaquePointer()), llvm::hash_value(prop.const_strides.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional CreateNdDescOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_shape") return prop.const_shape; if (name == "const_strides") return prop.const_strides; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void CreateNdDescOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_shape") { prop.const_shape = ::llvm::dyn_cast_or_null>(value); return; } if (name == "const_strides") { prop.const_strides = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void CreateNdDescOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_shape) attrs.append("const_shape", prop.const_shape); if (prop.const_strides) attrs.append("const_strides", prop.const_strides); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult CreateNdDescOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstShapeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_shape", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getConstStridesAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_strides", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult CreateNdDescOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.const_shape))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.const_strides))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void CreateNdDescOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.const_shape); writer.writeOptionalAttribute(prop.const_strides); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::std::optional<::llvm::ArrayRef> CreateNdDescOp::getConstShape() { auto attr = getConstShapeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> CreateNdDescOp::getConstStrides() { auto attr = getConstStridesAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } void CreateNdDescOp::setConstShape(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().const_shape; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue); else odsProp = nullptr; } void CreateNdDescOp::setConstStrides(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().const_strides; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue); else odsProp = nullptr; } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type tdesc, TypedValue source) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tdesc), std::forward(source)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, Type tdesc, TypedValue source) { return create(builder, builder.getLoc(), std::forward(tdesc), std::forward(source)); } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type tdesc, Value source, llvm::ArrayRef shape, llvm::ArrayRef strides) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tdesc), std::forward(source), std::forward(shape), std::forward(strides)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, Type tdesc, Value source, llvm::ArrayRef shape, llvm::ArrayRef strides) { return create(builder, builder.getLoc(), std::forward(tdesc), std::forward(source), std::forward(shape), std::forward(strides)); } void CreateNdDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { odsState.addOperands(source); odsState.addOperands(shape); odsState.addOperands(strides); ::llvm::copy(::llvm::ArrayRef({1, static_cast(shape.size()), static_cast(strides.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (const_shape) { odsState.getOrAddProperties().const_shape = const_shape; } if (const_strides) { odsState.getOrAddProperties().const_strides = const_strides; } odsState.addTypes(TensorDesc); } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(TensorDesc), std::forward(source), std::forward(shape), std::forward(strides), std::forward(const_shape), std::forward(const_strides)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type TensorDesc, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { return create(builder, builder.getLoc(), std::forward(TensorDesc), std::forward(source), std::forward(shape), std::forward(strides), std::forward(const_shape), std::forward(const_strides)); } void CreateNdDescOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { odsState.addOperands(source); odsState.addOperands(shape); odsState.addOperands(strides); ::llvm::copy(::llvm::ArrayRef({1, static_cast(shape.size()), static_cast(strides.size())}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (const_shape) { odsState.getOrAddProperties().const_shape = const_shape; } if (const_strides) { odsState.getOrAddProperties().const_strides = const_strides; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source), std::forward(shape), std::forward(strides), std::forward(const_shape), std::forward(const_strides)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::ValueRange shape, ::mlir::ValueRange strides, /*optional*/::mlir::DenseI64ArrayAttr const_shape, /*optional*/::mlir::DenseI64ArrayAttr const_strides) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source), std::forward(shape), std::forward(strides), std::forward(const_shape), std::forward(const_strides)); } void CreateNdDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CreateNdDescOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CreateNdDescOp CreateNdDescOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CreateNdDescOp CreateNdDescOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CreateNdDescOp::verifyInvariantsImpl() { auto tblgen_const_shape = getProperties().const_shape; (void)tblgen_const_shape; auto tblgen_const_strides = getProperties().const_strides; (void)tblgen_const_strides; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_shape, "const_shape"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_strides, "const_strides"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult CreateNdDescOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult CreateNdDescOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> shapeOperands; ::llvm::SMLoc shapeOperandsLoc; (void)shapeOperandsLoc; ::mlir::DenseI64ArrayAttr const_shapeAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> stridesOperands; ::llvm::SMLoc stridesOperandsLoc; (void)stridesOperandsLoc; ::mlir::DenseI64ArrayAttr const_stridesAttr; ::mlir::Type sourceRawType{}; ::llvm::ArrayRef<::mlir::Type> sourceTypes(&sourceRawType, 1); ::mlir::Type TensorDescRawType{}; ::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1); sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { if (parser.parseKeyword("shape")) return ::mlir::failure(); if (parser.parseColon()) return ::mlir::failure(); { shapeOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, shapeOperands, const_shapeAttr); if (odsResult) return ::mlir::failure(); if (const_shapeAttr) result.getOrAddProperties().const_shape = const_shapeAttr; } if (parser.parseComma()) return ::mlir::failure(); if (parser.parseKeyword("strides")) return ::mlir::failure(); if (parser.parseColon()) return ::mlir::failure(); { stridesOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, stridesOperands, const_stridesAttr); if (odsResult) return ::mlir::failure(); if (const_stridesAttr) result.getOrAddProperties().const_strides = const_stridesAttr; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); sourceRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); if (parser.parseType(TensorDescRawType)) return ::mlir::failure(); ::llvm::copy(::llvm::ArrayRef({1, static_cast(shapeOperands.size()), static_cast(stridesOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(TensorDescTypes); if (parser.resolveOperands(sourceOperands, sourceTypes, sourceOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(shapeOperands, odsBuildableType0, shapeOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(stridesOperands, odsBuildableType0, stridesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CreateNdDescOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); if (((!getShape().empty()) || (getConstShapeAttr()))) { _odsPrinter << ","; _odsPrinter << ' ' << "shape"; _odsPrinter << ' ' << ":"; _odsPrinter << ' '; printDynamicIndexList(_odsPrinter, *this, getShape(), getConstShapeAttr()); _odsPrinter << ","; _odsPrinter << ' ' << "strides"; _odsPrinter << ' ' << ":"; _odsPrinter << ' '; printDynamicIndexList(_odsPrinter, *this, getStrides(), getConstStridesAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("const_shape"); elidedAttrs.push_back("const_strides"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getSource().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; _odsPrinter << getTensorDesc().getType(); } void CreateNdDescOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::CreateNdDescOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::CreateNdDescOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::DpasMxOp definitions //===----------------------------------------------------------------------===// namespace detail { DpasMxOpGenericAdaptorBase::DpasMxOpGenericAdaptorBase(DpasMxOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DpasMxOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutA() { auto attr = getLayoutAAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutB() { auto attr = getLayoutBAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutCd() { auto attr = getLayoutCdAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutAScale() { auto attr = getLayoutAScaleAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOpGenericAdaptorBase::getLayoutBScale() { auto attr = getLayoutBScaleAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail DpasMxOpAdaptor::DpasMxOpAdaptor(DpasMxOp op) : DpasMxOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DpasMxOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a; auto tblgen_layout_a_scale = getProperties().layout_a_scale; (void)tblgen_layout_a_scale; auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b; auto tblgen_layout_b_scale = getProperties().layout_b_scale; (void)tblgen_layout_b_scale; auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd; if (tblgen_layout_a && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a)))) return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_a' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_b && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b)))) return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_b' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_cd && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_cd)))) return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_cd' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_a_scale && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a_scale)))) return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_a_scale' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_b_scale && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b_scale)))) return emitError(loc, "'xegpu.dpas_mx' op attribute 'layout_b_scale' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair DpasMxOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange DpasMxOp::getAccMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange DpasMxOp::getScaleAMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange DpasMxOp::getScaleBMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult DpasMxOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout_a; auto attr = dict.get("layout_a"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_a` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_a_scale; auto attr = dict.get("layout_a_scale"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_a_scale` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_b; auto attr = dict.get("layout_b"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_b` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_b_scale; auto attr = dict.get("layout_b_scale"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_b_scale` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_cd; auto attr = dict.get("layout_cd"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_cd` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute DpasMxOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout_a; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_a", propStorage)); } { const auto &propStorage = prop.layout_a_scale; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_a_scale", propStorage)); } { const auto &propStorage = prop.layout_b; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_b", propStorage)); } { const auto &propStorage = prop.layout_b_scale; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_b_scale", propStorage)); } { const auto &propStorage = prop.layout_cd; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_cd", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DpasMxOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout_a.getAsOpaquePointer()), llvm::hash_value(prop.layout_a_scale.getAsOpaquePointer()), llvm::hash_value(prop.layout_b.getAsOpaquePointer()), llvm::hash_value(prop.layout_b_scale.getAsOpaquePointer()), llvm::hash_value(prop.layout_cd.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional DpasMxOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout_a") return prop.layout_a; if (name == "layout_a_scale") return prop.layout_a_scale; if (name == "layout_b") return prop.layout_b; if (name == "layout_b_scale") return prop.layout_b_scale; if (name == "layout_cd") return prop.layout_cd; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void DpasMxOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout_a") { prop.layout_a = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_a_scale") { prop.layout_a_scale = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_b") { prop.layout_b = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_b_scale") { prop.layout_b_scale = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_cd") { prop.layout_cd = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void DpasMxOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout_a) attrs.append("layout_a", prop.layout_a); if (prop.layout_a_scale) attrs.append("layout_a_scale", prop.layout_a_scale); if (prop.layout_b) attrs.append("layout_b", prop.layout_b); if (prop.layout_b_scale) attrs.append("layout_b_scale", prop.layout_b_scale); if (prop.layout_cd) attrs.append("layout_cd", prop.layout_cd); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult DpasMxOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAScaleAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a_scale", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutBScaleAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b_scale", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutCdAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_cd", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult DpasMxOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a_scale))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b_scale))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_cd))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void DpasMxOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout_a); writer.writeOptionalAttribute(prop.layout_a_scale); writer.writeOptionalAttribute(prop.layout_b); writer.writeOptionalAttribute(prop.layout_b_scale); writer.writeOptionalAttribute(prop.layout_cd); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutA() { auto attr = getLayoutAAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutB() { auto attr = getLayoutBAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutCd() { auto attr = getLayoutCdAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutAScale() { auto attr = getLayoutAScaleAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasMxOp::getLayoutBScale() { auto attr = getLayoutBScaleAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void DpasMxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { odsState.addOperands(a); odsState.addOperands(b); if (acc) odsState.addOperands(acc); if (scale_a) odsState.addOperands(scale_a); if (scale_b) odsState.addOperands(scale_b); ::llvm::copy(::llvm::ArrayRef({1, 1, (acc ? 1 : 0), (scale_a ? 1 : 0), (scale_b ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout_a) { odsState.getOrAddProperties().layout_a = layout_a; } if (layout_b) { odsState.getOrAddProperties().layout_b = layout_b; } if (layout_cd) { odsState.getOrAddProperties().layout_cd = layout_cd; } if (layout_a_scale) { odsState.getOrAddProperties().layout_a_scale = layout_a_scale; } if (layout_b_scale) { odsState.getOrAddProperties().layout_b_scale = layout_b_scale; } odsState.addTypes(result); } DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(a), std::forward(b), std::forward(acc), std::forward(scale_a), std::forward(scale_b), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd), std::forward(layout_a_scale), std::forward(layout_b_scale)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { return create(builder, builder.getLoc(), std::forward(result), std::forward(a), std::forward(b), std::forward(acc), std::forward(scale_a), std::forward(scale_b), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd), std::forward(layout_a_scale), std::forward(layout_b_scale)); } void DpasMxOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { odsState.addOperands(a); odsState.addOperands(b); if (acc) odsState.addOperands(acc); if (scale_a) odsState.addOperands(scale_a); if (scale_b) odsState.addOperands(scale_b); ::llvm::copy(::llvm::ArrayRef({1, 1, (acc ? 1 : 0), (scale_a ? 1 : 0), (scale_b ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout_a) { odsState.getOrAddProperties().layout_a = layout_a; } if (layout_b) { odsState.getOrAddProperties().layout_b = layout_b; } if (layout_cd) { odsState.getOrAddProperties().layout_cd = layout_cd; } if (layout_a_scale) { odsState.getOrAddProperties().layout_a_scale = layout_a_scale; } if (layout_b_scale) { odsState.getOrAddProperties().layout_b_scale = layout_b_scale; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(a), std::forward(b), std::forward(acc), std::forward(scale_a), std::forward(scale_b), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd), std::forward(layout_a_scale), std::forward(layout_b_scale)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, /*optional*/::mlir::Value acc, /*optional*/::mlir::Value scale_a, /*optional*/::mlir::Value scale_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a_scale, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b_scale) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(a), std::forward(b), std::forward(acc), std::forward(scale_a), std::forward(scale_b), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd), std::forward(layout_a_scale), std::forward(layout_b_scale)); } void DpasMxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DpasMxOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DpasMxOp DpasMxOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasMxOp DpasMxOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DpasMxOp::verifyInvariantsImpl() { auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a; auto tblgen_layout_a_scale = getProperties().layout_a_scale; (void)tblgen_layout_a_scale; auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b; auto tblgen_layout_b_scale = getProperties().layout_b_scale; (void)tblgen_layout_b_scale; auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a, "layout_a"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b, "layout_b"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_cd, "layout_cd"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a_scale, "layout_a_scale"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b_scale, "layout_b_scale"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult DpasMxOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult DpasMxOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand aRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> aOperands(&aRawOperand, 1); ::llvm::SMLoc aOperandsLoc; (void)aOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand bRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> bOperands(&bRawOperand, 1); ::llvm::SMLoc bOperandsLoc; (void)bOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> scale_aOperands; ::llvm::SMLoc scale_aOperandsLoc; (void)scale_aOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> scale_bOperands; ::llvm::SMLoc scale_bOperandsLoc; (void)scale_bOperandsLoc; ::llvm::ArrayRef<::mlir::Type> allOperandTypes; ::llvm::ArrayRef<::mlir::Type> allResultTypes; aOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(aRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); bOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(bRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalKeyword("scale_a"))) { if (parser.parseEqual()) return ::mlir::failure(); { scale_aOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); scale_aOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalKeyword("scale_b"))) { if (parser.parseEqual()) return ::mlir::failure(); { scale_bOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); scale_bOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); ::mlir::FunctionType allOperand__allResult_functionType; if (parser.parseType(allOperand__allResult_functionType)) return ::mlir::failure(); allOperandTypes = allOperand__allResult_functionType.getInputs(); allResultTypes = allOperand__allResult_functionType.getResults(); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(accOperands.size()), static_cast(scale_aOperands.size()), static_cast(scale_bOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); result.addTypes(allResultTypes); if (parser.resolveOperands(::llvm::concat(aOperands, bOperands, accOperands, scale_aOperands, scale_bOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } void DpasMxOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getA(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getB(); if (getAcc()) { _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getAcc()) _odsPrinter << value; } if (getScaleA()) { _odsPrinter << ' ' << "scale_a"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getScaleA()) _odsPrinter << value; } if (getScaleB()) { _odsPrinter << ' ' << "scale_b"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; if (::mlir::Value value = getScaleB()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter.printFunctionalType(getOperation()->getOperandTypes(), getOperation()->getResultTypes()); } void DpasMxOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::DpasMxOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::DpasMxOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::DpasOp definitions //===----------------------------------------------------------------------===// namespace detail { DpasOpGenericAdaptorBase::DpasOpGenericAdaptorBase(DpasOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair DpasOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutA() { auto attr = getLayoutAAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutB() { auto attr = getLayoutBAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOpGenericAdaptorBase::getLayoutCd() { auto attr = getLayoutCdAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail DpasOpAdaptor::DpasOpAdaptor(DpasOp op) : DpasOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult DpasOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a; auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b; auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd; if (tblgen_layout_a && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_a)))) return emitError(loc, "'xegpu.dpas' op attribute 'layout_a' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_b && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_b)))) return emitError(loc, "'xegpu.dpas' op attribute 'layout_b' failed to satisfy constraint: DistributeLayoutAttr instance"); if (tblgen_layout_cd && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout_cd)))) return emitError(loc, "'xegpu.dpas' op attribute 'layout_cd' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair DpasOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange DpasOp::getAccMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult DpasOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout_a; auto attr = dict.get("layout_a"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_a` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_b; auto attr = dict.get("layout_b"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_b` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout_cd; auto attr = dict.get("layout_cd"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout_cd` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute DpasOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout_a; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_a", propStorage)); } { const auto &propStorage = prop.layout_b; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_b", propStorage)); } { const auto &propStorage = prop.layout_cd; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout_cd", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code DpasOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout_a.getAsOpaquePointer()), llvm::hash_value(prop.layout_b.getAsOpaquePointer()), llvm::hash_value(prop.layout_cd.getAsOpaquePointer())); } std::optional DpasOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout_a") return prop.layout_a; if (name == "layout_b") return prop.layout_b; if (name == "layout_cd") return prop.layout_cd; return std::nullopt; } void DpasOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout_a") { prop.layout_a = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_b") { prop.layout_b = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout_cd") { prop.layout_cd = ::llvm::dyn_cast_or_null>(value); return; } } void DpasOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout_a) attrs.append("layout_a", prop.layout_a); if (prop.layout_b) attrs.append("layout_b", prop.layout_b); if (prop.layout_cd) attrs.append("layout_cd", prop.layout_cd); } ::llvm::LogicalResult DpasOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_a", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutBAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_b", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutCdAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout_cd", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult DpasOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout_a))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_b))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout_cd))) return ::mlir::failure(); return ::mlir::success(); } void DpasOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout_a); writer.writeOptionalAttribute(prop.layout_b); writer.writeOptionalAttribute(prop.layout_cd); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutA() { auto attr = getLayoutAAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutB() { auto attr = getLayoutBAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> DpasOp::getLayoutCd() { auto attr = getLayoutCdAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void DpasOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (acc) odsState.addOperands(acc); if (layout_a) { odsState.getOrAddProperties().layout_a = layout_a; } if (layout_b) { odsState.getOrAddProperties().layout_b = layout_b; } if (layout_cd) { odsState.getOrAddProperties().layout_cd = layout_cd; } odsState.addTypes(result); } DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(acc), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(acc), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd)); } void DpasOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (acc) odsState.addOperands(acc); if (layout_a) { odsState.getOrAddProperties().layout_a = layout_a; } if (layout_b) { odsState.getOrAddProperties().layout_b = layout_b; } if (layout_cd) { odsState.getOrAddProperties().layout_cd = layout_cd; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(acc), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value acc, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_a, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_b, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout_cd) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(acc), std::forward(layout_a), std::forward(layout_b), std::forward(layout_cd)); } void DpasOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void DpasOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } DpasOp DpasOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } DpasOp DpasOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult DpasOp::verifyInvariantsImpl() { auto tblgen_layout_a = getProperties().layout_a; (void)tblgen_layout_a; auto tblgen_layout_b = getProperties().layout_b; (void)tblgen_layout_b; auto tblgen_layout_cd = getProperties().layout_cd; (void)tblgen_layout_cd; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_a, "layout_a"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_b, "layout_b"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout_cd, "layout_cd"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU9(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {lhs, rhs} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult DpasOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult DpasOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::llvm::SmallVector<::mlir::Type, 1> accTypes; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (::mlir::succeeded(parser.parseOptionalComma())) { { ::mlir::Type optionalType; ::mlir::OptionalParseResult parseResult = parser.parseOptionalType(optionalType); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accTypes.push_back(optionalType); } } } if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, accTypes, accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void DpasOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (getAcc()) { _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getAcc()) _odsPrinter << value; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } if (getAcc()) { _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << (getAcc() ? ::llvm::ArrayRef<::mlir::Type>(getAcc().getType()) : ::llvm::ArrayRef<::mlir::Type>()); } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void DpasOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::DpasOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::DpasOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::FenceOp definitions //===----------------------------------------------------------------------===// namespace detail { FenceOpGenericAdaptorBase::FenceOpGenericAdaptorBase(FenceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::xegpu::MemorySpace FenceOpGenericAdaptorBase::getMemoryKind() { auto attr = getMemoryKindAttr(); return attr.getValue(); } ::mlir::xegpu::FenceScope FenceOpGenericAdaptorBase::getFenceScope() { auto attr = getFenceScopeAttr(); return attr.getValue(); } } // namespace detail FenceOpAdaptor::FenceOpAdaptor(FenceOp op) : FenceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult FenceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_fence_scope = getProperties().fence_scope; (void)tblgen_fence_scope; if (!tblgen_fence_scope) return emitError(loc, "'xegpu.fence' op requires attribute 'fence_scope'"); auto tblgen_memory_kind = getProperties().memory_kind; (void)tblgen_memory_kind; if (!tblgen_memory_kind) return emitError(loc, "'xegpu.fence' op requires attribute 'memory_kind'"); if (tblgen_memory_kind && !((::llvm::isa<::mlir::xegpu::MemorySpaceAttr>(tblgen_memory_kind)))) return emitError(loc, "'xegpu.fence' op attribute 'memory_kind' failed to satisfy constraint: Describe the location of data described by a `TensorDesc`:\n Global device memory (`Global`) or Shared local memory (`SLM`)."); if (tblgen_fence_scope && !((::llvm::isa<::mlir::xegpu::FenceScopeAttr>(tblgen_fence_scope)))) return emitError(loc, "'xegpu.fence' op attribute 'fence_scope' failed to satisfy constraint: Describes the scope of fence.\n \"workgroup\" means that the scope is within each work group.\n \"gpu\" means the scope is across work groups within the gpu."); return ::mlir::success(); } ::llvm::LogicalResult FenceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.fence_scope; auto attr = dict.get("fence_scope"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `fence_scope` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.memory_kind; auto attr = dict.get("memory_kind"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `memory_kind` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute FenceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.fence_scope; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("fence_scope", propStorage)); } { const auto &propStorage = prop.memory_kind; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("memory_kind", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code FenceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.fence_scope.getAsOpaquePointer()), llvm::hash_value(prop.memory_kind.getAsOpaquePointer())); } std::optional FenceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "fence_scope") return prop.fence_scope; if (name == "memory_kind") return prop.memory_kind; return std::nullopt; } void FenceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "fence_scope") { prop.fence_scope = ::llvm::dyn_cast_or_null>(value); return; } if (name == "memory_kind") { prop.memory_kind = ::llvm::dyn_cast_or_null>(value); return; } } void FenceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.fence_scope) attrs.append("fence_scope", prop.fence_scope); if (prop.memory_kind) attrs.append("memory_kind", prop.memory_kind); } ::llvm::LogicalResult FenceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getFenceScopeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU6(attr, "fence_scope", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getMemoryKindAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU5(attr, "memory_kind", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult FenceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.fence_scope))) return ::mlir::failure(); if (::mlir::failed(reader.readAttribute(prop.memory_kind))) return ::mlir::failure(); return ::mlir::success(); } void FenceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.fence_scope); writer.writeAttribute(prop.memory_kind); } ::mlir::xegpu::MemorySpace FenceOp::getMemoryKind() { auto attr = getMemoryKindAttr(); return attr.getValue(); } ::mlir::xegpu::FenceScope FenceOp::getFenceScope() { auto attr = getFenceScopeAttr(); return attr.getValue(); } void FenceOp::setMemoryKind(::mlir::xegpu::MemorySpace attrValue) { getProperties().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void FenceOp::setFenceScope(::mlir::xegpu::FenceScope attrValue) { getProperties().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { odsState.getOrAddProperties().memory_kind = memory_kind; odsState.getOrAddProperties().fence_scope = fence_scope; } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(memory_kind), std::forward(fence_scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { return create(builder, builder.getLoc(), std::forward(memory_kind), std::forward(fence_scope)); } void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { odsState.getOrAddProperties().memory_kind = memory_kind; odsState.getOrAddProperties().fence_scope = fence_scope; assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(memory_kind), std::forward(fence_scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpaceAttr memory_kind, ::mlir::xegpu::FenceScopeAttr fence_scope) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(memory_kind), std::forward(fence_scope)); } void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { odsState.getOrAddProperties().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(odsBuilder.getContext(), memory_kind); odsState.getOrAddProperties().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(odsBuilder.getContext(), fence_scope); } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(memory_kind), std::forward(fence_scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { return create(builder, builder.getLoc(), std::forward(memory_kind), std::forward(fence_scope)); } void FenceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { odsState.getOrAddProperties().memory_kind = ::mlir::xegpu::MemorySpaceAttr::get(odsBuilder.getContext(), memory_kind); odsState.getOrAddProperties().fence_scope = ::mlir::xegpu::FenceScopeAttr::get(odsBuilder.getContext(), fence_scope); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(memory_kind), std::forward(fence_scope)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::xegpu::MemorySpace memory_kind, ::mlir::xegpu::FenceScope fence_scope) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(memory_kind), std::forward(fence_scope)); } void FenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void FenceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } FenceOp FenceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FenceOp FenceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult FenceOp::verifyInvariantsImpl() { auto tblgen_fence_scope = getProperties().fence_scope; (void)tblgen_fence_scope; if (!tblgen_fence_scope) return emitOpError("requires attribute 'fence_scope'"); auto tblgen_memory_kind = getProperties().memory_kind; (void)tblgen_memory_kind; if (!tblgen_memory_kind) return emitOpError("requires attribute 'memory_kind'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU5(*this, tblgen_memory_kind, "memory_kind"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU6(*this, tblgen_fence_scope, "fence_scope"))) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult FenceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult FenceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::xegpu::MemorySpaceAttr memory_kindAttr; ::mlir::xegpu::FenceScopeAttr fence_scopeAttr; if (parser.parseKeyword("memory_kind")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(memory_kindAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (memory_kindAttr) result.getOrAddProperties().memory_kind = memory_kindAttr; if (parser.parseComma()) return ::mlir::failure(); if (parser.parseKeyword("fence_scope")) return ::mlir::failure(); if (parser.parseEqual()) return ::mlir::failure(); if (parser.parseCustomAttributeWithFallback(fence_scopeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (fence_scopeAttr) result.getOrAddProperties().fence_scope = fence_scopeAttr; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } return ::mlir::success(); } void FenceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' ' << "memory_kind"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getMemoryKindAttr()); _odsPrinter << ","; _odsPrinter << ' ' << "fence_scope"; _odsPrinter << ' ' << "="; _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getFenceScopeAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("memory_kind"); elidedAttrs.push_back("fence_scope"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::FenceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::FenceOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::InitNbarrierOp definitions //===----------------------------------------------------------------------===// namespace detail { InitNbarrierOpGenericAdaptorBase::InitNbarrierOpGenericAdaptorBase(InitNbarrierOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail InitNbarrierOpAdaptor::InitNbarrierOpAdaptor(InitNbarrierOp op) : InitNbarrierOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult InitNbarrierOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult InitNbarrierOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute InitNbarrierOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code InitNbarrierOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional InitNbarrierOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void InitNbarrierOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void InitNbarrierOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult InitNbarrierOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void InitNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { odsState.addOperands(nbarrier_id); odsState.addOperands(participant_thread_num); odsState.addTypes(result); } InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(nbarrier_id), std::forward(participant_thread_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { return create(builder, builder.getLoc(), std::forward(result), std::forward(nbarrier_id), std::forward(participant_thread_num)); } void InitNbarrierOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { odsState.addOperands(nbarrier_id); odsState.addOperands(participant_thread_num); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(nbarrier_id), std::forward(participant_thread_num)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier_id, ::mlir::Value participant_thread_num) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(nbarrier_id), std::forward(participant_thread_num)); } void InitNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void InitNbarrierOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } InitNbarrierOp InitNbarrierOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InitNbarrierOp InitNbarrierOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult InitNbarrierOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU11(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult InitNbarrierOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult InitNbarrierOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand nbarrier_idRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrier_idOperands(&nbarrier_idRawOperand, 1); ::llvm::SMLoc nbarrier_idOperandsLoc; (void)nbarrier_idOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand participant_thread_numRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> participant_thread_numOperands(&participant_thread_numRawOperand, 1); ::llvm::SMLoc participant_thread_numOperandsLoc; (void)participant_thread_numOperandsLoc; ::mlir::Type nbarrier_idRawType{}; ::llvm::ArrayRef<::mlir::Type> nbarrier_idTypes(&nbarrier_idRawType, 1); ::mlir::Type participant_thread_numRawType{}; ::llvm::ArrayRef<::mlir::Type> participant_thread_numTypes(&participant_thread_numRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); nbarrier_idOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nbarrier_idRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); participant_thread_numOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(participant_thread_numRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::IntegerType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); nbarrier_idRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::IntegerType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); participant_thread_numRawType = type; } if (parser.parseArrow()) return ::mlir::failure(); if (parser.parseType(resultRawType)) return ::mlir::failure(); result.addTypes(resultTypes); if (parser.resolveOperands(nbarrier_idOperands, nbarrier_idTypes, nbarrier_idOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(participant_thread_numOperands, participant_thread_numTypes, participant_thread_numOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void InitNbarrierOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getNbarrierId(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getParticipantThreadNum(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getNbarrierId().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getParticipantThreadNum().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::IntegerType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "->"; _odsPrinter << ' '; _odsPrinter << getResult().getType(); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::InitNbarrierOp) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::LoadGatherOp definitions //===----------------------------------------------------------------------===// namespace detail { LoadGatherOpGenericAdaptorBase::LoadGatherOpGenericAdaptorBase(LoadGatherOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional LoadGatherOpGenericAdaptorBase::getChunkSize() { auto attr = getChunkSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadGatherOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail LoadGatherOpAdaptor::LoadGatherOpAdaptor(LoadGatherOp op) : LoadGatherOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LoadGatherOpAdaptor::verify(::mlir::Location loc) { auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size; auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_chunk_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_chunk_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_chunk_size).getType().isSignlessInteger(64))))) return emitError(loc, "'xegpu.load' op attribute 'chunk_size' failed to satisfy constraint: 64-bit signless integer attribute"); if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.load' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.load' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.load' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.load' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } ::llvm::LogicalResult LoadGatherOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.chunk_size; auto attr = dict.get("chunk_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LoadGatherOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.chunk_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("chunk_size", propStorage)); } { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LoadGatherOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.chunk_size.getAsOpaquePointer()), llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional LoadGatherOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "chunk_size") return prop.chunk_size; if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; return std::nullopt; } void LoadGatherOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "chunk_size") { prop.chunk_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void LoadGatherOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.chunk_size) attrs.append("chunk_size", prop.chunk_size); if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult LoadGatherOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getChunkSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "chunk_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LoadGatherOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.chunk_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void LoadGatherOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.chunk_size); writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); } ::std::optional LoadGatherOp::getChunkSize() { auto attr = getChunkSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadGatherOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadGatherOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void LoadGatherOp::setChunkSize(::std::optional attrValue) { auto &odsProp = getProperties().chunk_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } void LoadGatherOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LoadGatherOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LoadGatherOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value source, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value source, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) { return create(builder, builder.getLoc(), std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint)); } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value source, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value source, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadGatherOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(source); odsState.addOperands(offsets); odsState.addOperands(mask); if (chunk_size) { odsState.getOrAddProperties().chunk_size = chunk_size; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(value); } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadGatherOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(source); odsState.addOperands(offsets); odsState.addOperands(mask); if (chunk_size) { odsState.getOrAddProperties().chunk_size = chunk_size; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadGatherOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LoadGatherOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LoadGatherOp LoadGatherOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadGatherOp LoadGatherOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult LoadGatherOp::verifyInvariantsImpl() { auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size; auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_chunk_size, "chunk_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult LoadGatherOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult LoadGatherOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); offsetsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(offsetsRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); maskOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(maskRawOperand)) return ::mlir::failure(); if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } result.addTypes(valueTypes); if (parser.resolveOperands(::llvm::concat(sourceOperands, offsetsOperands, maskOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult LoadGatherOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.chunk_size; auto chunk_sizeAttrName = ::mlir::StringAttr::get(ctx, "chunk_size"); auto attr = dict.get(chunk_sizeAttrName); usedKeys.insert(chunk_sizeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for chunk_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void LoadGatherOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); _odsPrinter << "["; _odsPrinter << getOffsets(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMask(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void LoadGatherOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadGatherOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadGatherOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::LoadMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { LoadMatrixOpGenericAdaptorBase::LoadMatrixOpGenericAdaptorBase(LoadMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LoadMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::ArrayRef LoadMatrixOpGenericAdaptorBase::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional LoadMatrixOpGenericAdaptorBase::getSubgroupBlockIo() { auto attr = getSubgroupBlockIoAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadMatrixOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail LoadMatrixOpAdaptor::LoadMatrixOpAdaptor(LoadMatrixOp op) : LoadMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LoadMatrixOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitError(loc, "'xegpu.load_matrix' op requires attribute 'const_offsets'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io; if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets)))) return emitError(loc, "'xegpu.load_matrix' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_subgroup_block_io && !((::llvm::isa<::mlir::UnitAttr>(tblgen_subgroup_block_io)))) return emitError(loc, "'xegpu.load_matrix' op attribute 'subgroup_block_io' failed to satisfy constraint: unit attribute"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.load_matrix' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair LoadMatrixOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange LoadMatrixOp::getOffsetsMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult LoadMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_offsets; auto attr = dict.get("const_offsets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.subgroup_block_io; auto attr = dict.get("subgroup_block_io"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LoadMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_offsets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_offsets", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.subgroup_block_io; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("subgroup_block_io", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LoadMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_offsets.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer()), llvm::hash_value(prop.subgroup_block_io.getAsOpaquePointer())); } std::optional LoadMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_offsets") return prop.const_offsets; if (name == "layout") return prop.layout; if (name == "subgroup_block_io") return prop.subgroup_block_io; return std::nullopt; } void LoadMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_offsets") { prop.const_offsets = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "subgroup_block_io") { prop.subgroup_block_io = ::llvm::dyn_cast_or_null>(value); return; } } void LoadMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets); if (prop.layout) attrs.append("layout", prop.layout); if (prop.subgroup_block_io) attrs.append("subgroup_block_io", prop.subgroup_block_io); } ::llvm::LogicalResult LoadMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getSubgroupBlockIoAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "subgroup_block_io", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LoadMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.const_offsets))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.subgroup_block_io))) return ::mlir::failure(); return ::mlir::success(); } void LoadMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.const_offsets); writer.writeOptionalAttribute(prop.layout); writer.writeOptionalAttribute(prop.subgroup_block_io); } ::llvm::ArrayRef LoadMatrixOp::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional LoadMatrixOp::getSubgroupBlockIo() { auto attr = getSubgroupBlockIoAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadMatrixOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void LoadMatrixOp::setConstOffsets(::llvm::ArrayRef attrValue) { getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue); } void LoadMatrixOp::setSubgroupBlockIo(bool attrValue) { auto &odsProp = getProperties().subgroup_block_io; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type res, TypedValue mem_desc, llvm::ArrayRef offsets, DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, Type res, TypedValue mem_desc, llvm::ArrayRef offsets, DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(layout)); } void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(res); } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(res); } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(res), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void LoadMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void LoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LoadMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LoadMatrixOp LoadMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadMatrixOp LoadMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult LoadMatrixOp::verifyInvariantsImpl() { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_subgroup_block_io, "subgroup_block_io"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin()))) && (getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {mem_desc, res} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult LoadMatrixOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult LoadMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand mem_descRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mem_descOperands(&mem_descRawOperand, 1); ::llvm::SMLoc mem_descOperandsLoc; (void)mem_descOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands; ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::DenseI64ArrayAttr const_offsetsAttr; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; ::llvm::SmallVector<::mlir::Type, 1> allResultTypes; mem_descOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mem_descRawOperand)) return ::mlir::failure(); { offsetsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().const_offsets = const_offsetsAttr; } if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); if (parser.parseTypeList(allResultTypes)) return ::mlir::failure(); result.addTypes(allResultTypes); if (parser.resolveOperands(::llvm::concat(mem_descOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult LoadMatrixOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.subgroup_block_io; auto subgroup_block_ioAttrName = ::mlir::StringAttr::get(ctx, "subgroup_block_io"); auto attr = dict.get(subgroup_block_ioAttrName); usedKeys.insert(subgroup_block_ioAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for subgroup_block_io in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void LoadMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getMemDesc(); printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; elidedProps.push_back("const_offsets"); _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("const_offsets"); _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; _odsPrinter << getOperation()->getResultTypes(); } void LoadMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadMatrixOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadMatrixOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::LoadNdOp definitions //===----------------------------------------------------------------------===// namespace detail { LoadNdOpGenericAdaptorBase::LoadNdOpGenericAdaptorBase(LoadNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LoadNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::ArrayRef LoadNdOpGenericAdaptorBase::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional LoadNdOpGenericAdaptorBase::getPacked() { auto attr = getPackedAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> LoadNdOpGenericAdaptorBase::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadNdOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail LoadNdOpAdaptor::LoadNdOpAdaptor(LoadNdOp op) : LoadNdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LoadNdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitError(loc, "'xegpu.load_nd' op requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_packed = getProperties().packed; (void)tblgen_packed; auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets)))) return emitError(loc, "'xegpu.load_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_packed && !((::llvm::isa<::mlir::UnitAttr>(tblgen_packed)))) return emitError(loc, "'xegpu.load_nd' op attribute 'packed' failed to satisfy constraint: unit attribute"); if (tblgen_transpose && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_transpose)))) return emitError(loc, "'xegpu.load_nd' op attribute 'transpose' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.load_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.load_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.load_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.load_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair LoadNdOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange LoadNdOp::getOffsetsMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult LoadNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_offsets; auto attr = dict.get("const_offsets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.packed; auto attr = dict.get("packed"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `packed` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transpose; auto attr = dict.get("transpose"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transpose` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LoadNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_offsets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_offsets", propStorage)); } { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.packed; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("packed", propStorage)); } { const auto &propStorage = prop.transpose; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("transpose", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LoadNdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_offsets.getAsOpaquePointer()), llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer()), llvm::hash_value(prop.packed.getAsOpaquePointer()), llvm::hash_value(prop.transpose.getAsOpaquePointer())); } std::optional LoadNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_offsets") return prop.const_offsets; if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; if (name == "packed") return prop.packed; if (name == "transpose") return prop.transpose; return std::nullopt; } void LoadNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_offsets") { prop.const_offsets = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "packed") { prop.packed = ::llvm::dyn_cast_or_null>(value); return; } if (name == "transpose") { prop.transpose = ::llvm::dyn_cast_or_null>(value); return; } } void LoadNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets); if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); if (prop.packed) attrs.append("packed", prop.packed); if (prop.transpose) attrs.append("transpose", prop.transpose); } ::llvm::LogicalResult LoadNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getPackedAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "packed", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getTransposeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "transpose", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LoadNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.const_offsets))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.packed))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.transpose))) return ::mlir::failure(); return ::mlir::success(); } void LoadNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.const_offsets); writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); writer.writeOptionalAttribute(prop.packed); writer.writeOptionalAttribute(prop.transpose); } ::llvm::ArrayRef LoadNdOp::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional LoadNdOp::getPacked() { auto attr = getPackedAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::llvm::ArrayRef> LoadNdOp::getTranspose() { auto attr = getTransposeAttr(); return attr ? ::std::optional<::llvm::ArrayRef>(attr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> LoadNdOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> LoadNdOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void LoadNdOp::setConstOffsets(::llvm::ArrayRef attrValue) { getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue); } void LoadNdOp::setPacked(bool attrValue) { auto &odsProp = getProperties().packed; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } void LoadNdOp::setTranspose(::std::optional<::llvm::ArrayRef> attrValue) { auto &odsProp = getProperties().transpose; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(*attrValue); else odsProp = nullptr; } void LoadNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LoadNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void LoadNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Type value, Value TensorDesc, ArrayRef offsets, UnitAttr packed, DenseI64ArrayAttr transpose, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Type value, Value TensorDesc, ArrayRef offsets, UnitAttr packed, DenseI64ArrayAttr transpose, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (packed) { odsState.getOrAddProperties().packed = packed; } if (transpose) { odsState.getOrAddProperties().transpose = transpose; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(value); } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (packed) { odsState.getOrAddProperties().packed = packed; } if (transpose) { odsState.getOrAddProperties().transpose = transpose; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (packed) { odsState.getOrAddProperties().packed = packed; } if (transpose) { odsState.getOrAddProperties().transpose = transpose; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(value); } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (packed) { odsState.getOrAddProperties().packed = packed; } if (transpose) { odsState.getOrAddProperties().transpose = transpose; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr packed, /*optional*/::mlir::DenseI64ArrayAttr transpose, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(packed), std::forward(transpose), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void LoadNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LoadNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LoadNdOp LoadNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadNdOp LoadNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult LoadNdOp::verifyInvariantsImpl() { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_packed = getProperties().packed; (void)tblgen_packed; auto tblgen_transpose = getProperties().transpose; (void)tblgen_transpose; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_packed, "packed"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_transpose, "transpose"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin())))))) return emitOpError("failed to verify that all of {value, TensorDesc} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult LoadNdOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult LoadNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc; (void)TensorDescOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands; ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::DenseI64ArrayAttr const_offsetsAttr; ::mlir::Type TensorDescRawType{}; ::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1); ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); TensorDescOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(TensorDescRawOperand)) return ::mlir::failure(); { offsetsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().const_offsets = const_offsetsAttr; } if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseType(TensorDescRawType)) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(valueTypes); if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult LoadNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.packed; auto packedAttrName = ::mlir::StringAttr::get(ctx, "packed"); auto attr = dict.get(packedAttrName); usedKeys.insert(packedAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for packed in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `packed` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.transpose; auto transposeAttrName = ::mlir::StringAttr::get(ctx, "transpose"); auto attr = dict.get(transposeAttrName); usedKeys.insert(transposeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for transpose in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `transpose` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void LoadNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensorDesc(); printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; elidedProps.push_back("const_offsets"); _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("const_offsets"); _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getTensorDesc().getType(); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void LoadNdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::LoadNdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::LoadNdOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::NbarrierArriveOp definitions //===----------------------------------------------------------------------===// namespace detail { NbarrierArriveOpGenericAdaptorBase::NbarrierArriveOpGenericAdaptorBase(NbarrierArriveOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail NbarrierArriveOpAdaptor::NbarrierArriveOpAdaptor(NbarrierArriveOp op) : NbarrierArriveOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult NbarrierArriveOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult NbarrierArriveOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute NbarrierArriveOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code NbarrierArriveOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional NbarrierArriveOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void NbarrierArriveOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void NbarrierArriveOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult NbarrierArriveOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void NbarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value nbarrier) { odsState.addOperands(nbarrier); } NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value nbarrier) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(nbarrier)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value nbarrier) { return create(builder, builder.getLoc(), std::forward(nbarrier)); } void NbarrierArriveOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { odsState.addOperands(nbarrier); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(nbarrier)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(nbarrier)); } void NbarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void NbarrierArriveOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } NbarrierArriveOp NbarrierArriveOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierArriveOp NbarrierArriveOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult NbarrierArriveOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult NbarrierArriveOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult NbarrierArriveOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand nbarrierRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrierOperands(&nbarrierRawOperand, 1); ::llvm::SMLoc nbarrierOperandsLoc; (void)nbarrierOperandsLoc; ::mlir::Type nbarrierRawType{}; ::llvm::ArrayRef<::mlir::Type> nbarrierTypes(&nbarrierRawType, 1); nbarrierOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nbarrierRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseType(nbarrierRawType)) return ::mlir::failure(); if (parser.resolveOperands(nbarrierOperands, nbarrierTypes, nbarrierOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void NbarrierArriveOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getNbarrier(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getNbarrier().getType(); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::NbarrierArriveOp) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::NbarrierWaitOp definitions //===----------------------------------------------------------------------===// namespace detail { NbarrierWaitOpGenericAdaptorBase::NbarrierWaitOpGenericAdaptorBase(NbarrierWaitOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail NbarrierWaitOpAdaptor::NbarrierWaitOpAdaptor(NbarrierWaitOp op) : NbarrierWaitOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult NbarrierWaitOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult NbarrierWaitOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute NbarrierWaitOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code NbarrierWaitOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional NbarrierWaitOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void NbarrierWaitOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void NbarrierWaitOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult NbarrierWaitOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void NbarrierWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value nbarrier) { odsState.addOperands(nbarrier); } NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value nbarrier) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(nbarrier)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value nbarrier) { return create(builder, builder.getLoc(), std::forward(nbarrier)); } void NbarrierWaitOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { odsState.addOperands(nbarrier); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(nbarrier)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value nbarrier) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(nbarrier)); } void NbarrierWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void NbarrierWaitOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } NbarrierWaitOp NbarrierWaitOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } NbarrierWaitOp NbarrierWaitOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult NbarrierWaitOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult NbarrierWaitOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult NbarrierWaitOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand nbarrierRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> nbarrierOperands(&nbarrierRawOperand, 1); ::llvm::SMLoc nbarrierOperandsLoc; (void)nbarrierOperandsLoc; ::mlir::Type nbarrierRawType{}; ::llvm::ArrayRef<::mlir::Type> nbarrierTypes(&nbarrierRawType, 1); nbarrierOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(nbarrierRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseType(nbarrierRawType)) return ::mlir::failure(); if (parser.resolveOperands(nbarrierOperands, nbarrierTypes, nbarrierOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void NbarrierWaitOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getNbarrier(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getNbarrier().getType(); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::NbarrierWaitOp) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::PrefetchNdOp definitions //===----------------------------------------------------------------------===// namespace detail { PrefetchNdOpGenericAdaptorBase::PrefetchNdOpGenericAdaptorBase(PrefetchNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair PrefetchNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::ArrayRef PrefetchNdOpGenericAdaptorBase::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchNdOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail PrefetchNdOpAdaptor::PrefetchNdOpAdaptor(PrefetchNdOp op) : PrefetchNdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult PrefetchNdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitError(loc, "'xegpu.prefetch_nd' op requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets)))) return emitError(loc, "'xegpu.prefetch_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.prefetch_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.prefetch_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair PrefetchNdOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 1) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange PrefetchNdOp::getOffsetsMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult PrefetchNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_offsets; auto attr = dict.get("const_offsets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute PrefetchNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_offsets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_offsets", propStorage)); } { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code PrefetchNdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_offsets.getAsOpaquePointer()), llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional PrefetchNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_offsets") return prop.const_offsets; if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; return std::nullopt; } void PrefetchNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_offsets") { prop.const_offsets = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void PrefetchNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets); if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult PrefetchNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult PrefetchNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.const_offsets))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void PrefetchNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.const_offsets); writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); } ::llvm::ArrayRef PrefetchNdOp::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchNdOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchNdOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void PrefetchNdOp::setConstOffsets(::llvm::ArrayRef attrValue) { getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue); } void PrefetchNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void PrefetchNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void PrefetchNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value TensorDesc, ArrayRef offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(TensorDesc), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Value TensorDesc, ArrayRef offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(TensorDesc), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void PrefetchNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void PrefetchNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void PrefetchNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } PrefetchNdOp PrefetchNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchNdOp PrefetchNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult PrefetchNdOp::verifyInvariantsImpl() { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult PrefetchNdOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult PrefetchNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc; (void)TensorDescOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands; ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::DenseI64ArrayAttr const_offsetsAttr; ::mlir::Type TensorDescRawType{}; ::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1); TensorDescOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(TensorDescRawOperand)) return ::mlir::failure(); { offsetsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().const_offsets = const_offsetsAttr; } if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseType(TensorDescRawType)) return ::mlir::failure(); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult PrefetchNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void PrefetchNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTensorDesc(); printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; elidedProps.push_back("const_offsets"); _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("const_offsets"); _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getTensorDesc().getType(); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::PrefetchNdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::PrefetchNdOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::PrefetchOp definitions //===----------------------------------------------------------------------===// namespace detail { PrefetchOpGenericAdaptorBase::PrefetchOpGenericAdaptorBase(PrefetchOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional PrefetchOpGenericAdaptorBase::getOffsetAlignByte() { auto attr = getOffsetAlignByteAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail PrefetchOpAdaptor::PrefetchOpAdaptor(PrefetchOp op) : PrefetchOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult PrefetchOpAdaptor::verify(::mlir::Location loc) { auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_offset_align_byte = getProperties().offset_align_byte; (void)tblgen_offset_align_byte; if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.prefetch' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.prefetch' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.prefetch' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_offset_align_byte && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_offset_align_byte))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_offset_align_byte).getType().isSignlessInteger(64))))) return emitError(loc, "'xegpu.prefetch' op attribute 'offset_align_byte' failed to satisfy constraint: 64-bit signless integer attribute"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.prefetch' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } ::llvm::LogicalResult PrefetchOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.offset_align_byte; auto attr = dict.get("offset_align_byte"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `offset_align_byte` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute PrefetchOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.offset_align_byte; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("offset_align_byte", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code PrefetchOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer()), llvm::hash_value(prop.offset_align_byte.getAsOpaquePointer())); } std::optional PrefetchOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; if (name == "offset_align_byte") return prop.offset_align_byte; return std::nullopt; } void PrefetchOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "offset_align_byte") { prop.offset_align_byte = ::llvm::dyn_cast_or_null>(value); return; } } void PrefetchOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); if (prop.offset_align_byte) attrs.append("offset_align_byte", prop.offset_align_byte); } ::llvm::LogicalResult PrefetchOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getOffsetAlignByteAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "offset_align_byte", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult PrefetchOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.offset_align_byte))) return ::mlir::failure(); return ::mlir::success(); } void PrefetchOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); writer.writeOptionalAttribute(prop.offset_align_byte); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> PrefetchOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional PrefetchOp::getOffsetAlignByte() { auto attr = getOffsetAlignByteAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> PrefetchOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void PrefetchOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void PrefetchOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void PrefetchOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void PrefetchOp::setOffsetAlignByte(::std::optional attrValue) { auto &odsProp = getProperties().offset_align_byte; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } void PrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(source); odsState.addOperands(offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (offset_align_byte) { odsState.getOrAddProperties().offset_align_byte = offset_align_byte; } if (layout) { odsState.getOrAddProperties().layout = layout; } } PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(source), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(offset_align_byte), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(source), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(offset_align_byte), std::forward(layout)); } void PrefetchOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(source); odsState.addOperands(offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (offset_align_byte) { odsState.getOrAddProperties().offset_align_byte = offset_align_byte; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(offset_align_byte), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source, ::mlir::Value offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::IntegerAttr offset_align_byte, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(offset_align_byte), std::forward(layout)); } void PrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void PrefetchOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } PrefetchOp PrefetchOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } PrefetchOp PrefetchOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult PrefetchOp::verifyInvariantsImpl() { auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_offset_align_byte = getProperties().offset_align_byte; (void)tblgen_offset_align_byte; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_offset_align_byte, "offset_align_byte"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult PrefetchOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult PrefetchOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); offsetsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(offsetsRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.resolveOperands(::llvm::concat(sourceOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult PrefetchOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.offset_align_byte; auto offset_align_byteAttrName = ::mlir::StringAttr::get(ctx, "offset_align_byte"); auto attr = dict.get(offset_align_byteAttrName); usedKeys.insert(offset_align_byteAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for offset_align_byte in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `offset_align_byte` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void PrefetchOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); _odsPrinter << "["; _odsPrinter << getOffsets(); _odsPrinter << "]"; ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::PrefetchOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::PrefetchOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::StoreMatrixOp definitions //===----------------------------------------------------------------------===// namespace detail { StoreMatrixOpGenericAdaptorBase::StoreMatrixOpGenericAdaptorBase(StoreMatrixOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair StoreMatrixOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::ArrayRef StoreMatrixOpGenericAdaptorBase::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional StoreMatrixOpGenericAdaptorBase::getSubgroupBlockIo() { auto attr = getSubgroupBlockIoAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreMatrixOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail StoreMatrixOpAdaptor::StoreMatrixOpAdaptor(StoreMatrixOp op) : StoreMatrixOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult StoreMatrixOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitError(loc, "'xegpu.store_matrix' op requires attribute 'const_offsets'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io; if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets)))) return emitError(loc, "'xegpu.store_matrix' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_subgroup_block_io && !((::llvm::isa<::mlir::UnitAttr>(tblgen_subgroup_block_io)))) return emitError(loc, "'xegpu.store_matrix' op attribute 'subgroup_block_io' failed to satisfy constraint: unit attribute"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.store_matrix' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair StoreMatrixOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange StoreMatrixOp::getOffsetsMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult StoreMatrixOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_offsets; auto attr = dict.get("const_offsets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.subgroup_block_io; auto attr = dict.get("subgroup_block_io"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute StoreMatrixOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_offsets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_offsets", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.subgroup_block_io; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("subgroup_block_io", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code StoreMatrixOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_offsets.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer()), llvm::hash_value(prop.subgroup_block_io.getAsOpaquePointer())); } std::optional StoreMatrixOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_offsets") return prop.const_offsets; if (name == "layout") return prop.layout; if (name == "subgroup_block_io") return prop.subgroup_block_io; return std::nullopt; } void StoreMatrixOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_offsets") { prop.const_offsets = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "subgroup_block_io") { prop.subgroup_block_io = ::llvm::dyn_cast_or_null>(value); return; } } void StoreMatrixOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets); if (prop.layout) attrs.append("layout", prop.layout); if (prop.subgroup_block_io) attrs.append("subgroup_block_io", prop.subgroup_block_io); } ::llvm::LogicalResult StoreMatrixOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getSubgroupBlockIoAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(attr, "subgroup_block_io", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult StoreMatrixOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.const_offsets))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.subgroup_block_io))) return ::mlir::failure(); return ::mlir::success(); } void StoreMatrixOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.const_offsets); writer.writeOptionalAttribute(prop.layout); writer.writeOptionalAttribute(prop.subgroup_block_io); } ::llvm::ArrayRef StoreMatrixOp::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional StoreMatrixOp::getSubgroupBlockIo() { auto attr = getSubgroupBlockIoAttr(); return attr ? ::std::optional(attr != nullptr) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreMatrixOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void StoreMatrixOp::setConstOffsets(::llvm::ArrayRef attrValue) { getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue); } void StoreMatrixOp::setSubgroupBlockIo(bool attrValue) { auto &odsProp = getProperties().subgroup_block_io; if (attrValue) odsProp = ((attrValue) ? ::mlir::Builder((*this)->getContext()).getUnitAttr() : nullptr); else odsProp = nullptr; } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value data, TypedValue mem_desc, llvm::ArrayRef offsets, DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, Value data, TypedValue mem_desc, llvm::ArrayRef offsets, DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(layout)); } void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(data); odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(data); odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(data); odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void StoreMatrixOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(data); odsState.addOperands(mem_desc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (subgroup_block_io) { odsState.getOrAddProperties().subgroup_block_io = subgroup_block_io; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value data, ::mlir::Value mem_desc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::UnitAttr subgroup_block_io, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(data), std::forward(mem_desc), std::forward(offsets), std::forward(const_offsets), std::forward(subgroup_block_io), std::forward(layout)); } void StoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void StoreMatrixOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } StoreMatrixOp StoreMatrixOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreMatrixOp StoreMatrixOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult StoreMatrixOp::verifyInvariantsImpl() { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'"); auto tblgen_layout = getProperties().layout; (void)tblgen_layout; auto tblgen_subgroup_block_io = getProperties().subgroup_block_io; (void)tblgen_subgroup_block_io; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU8(*this, tblgen_subgroup_block_io, "subgroup_block_io"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU6(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin())))))) return emitOpError("failed to verify that all of {mem_desc, data} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult StoreMatrixOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult StoreMatrixOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand dataRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> dataOperands(&dataRawOperand, 1); ::llvm::SMLoc dataOperandsLoc; (void)dataOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand mem_descRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> mem_descOperands(&mem_descRawOperand, 1); ::llvm::SMLoc mem_descOperandsLoc; (void)mem_descOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands; ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::DenseI64ArrayAttr const_offsetsAttr; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; dataOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(dataRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); mem_descOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(mem_descRawOperand)) return ::mlir::failure(); { offsetsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().const_offsets = const_offsetsAttr; } if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.resolveOperands(::llvm::concat(dataOperands, mem_descOperands, offsetsOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult StoreMatrixOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.subgroup_block_io; auto subgroup_block_ioAttrName = ::mlir::StringAttr::get(ctx, "subgroup_block_io"); auto attr = dict.get(subgroup_block_ioAttrName); usedKeys.insert(subgroup_block_ioAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for subgroup_block_io in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `subgroup_block_io` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void StoreMatrixOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getData(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMemDesc(); printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; elidedProps.push_back("const_offsets"); _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("const_offsets"); _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); } void StoreMatrixOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreMatrixOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreMatrixOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::StoreNdOp definitions //===----------------------------------------------------------------------===// namespace detail { StoreNdOpGenericAdaptorBase::StoreNdOpGenericAdaptorBase(StoreNdOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair StoreNdOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::llvm::ArrayRef StoreNdOpGenericAdaptorBase::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreNdOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail StoreNdOpAdaptor::StoreNdOpAdaptor(StoreNdOp op) : StoreNdOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult StoreNdOpAdaptor::verify(::mlir::Location loc) { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitError(loc, "'xegpu.store_nd' op requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_const_offsets && !((::llvm::isa<::mlir::DenseI64ArrayAttr>(tblgen_const_offsets)))) return emitError(loc, "'xegpu.store_nd' op attribute 'const_offsets' failed to satisfy constraint: i64 dense array attribute"); if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.store_nd' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.store_nd' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.store_nd' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.store_nd' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } std::pair StoreNdOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 2) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange StoreNdOp::getOffsetsMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult StoreNdOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.const_offsets; auto attr = dict.get("const_offsets"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `const_offsets` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute StoreNdOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.const_offsets; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("const_offsets", propStorage)); } { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code StoreNdOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.const_offsets.getAsOpaquePointer()), llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional StoreNdOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "const_offsets") return prop.const_offsets; if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; return std::nullopt; } void StoreNdOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "const_offsets") { prop.const_offsets = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void StoreNdOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.const_offsets) attrs.append("const_offsets", prop.const_offsets); if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult StoreNdOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getConstOffsetsAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(attr, "const_offsets", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult StoreNdOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.const_offsets))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void StoreNdOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.const_offsets); writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); } ::llvm::ArrayRef StoreNdOp::getConstOffsets() { auto attr = getConstOffsetsAttr(); return attr; } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreNdOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreNdOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void StoreNdOp::setConstOffsets(::llvm::ArrayRef attrValue) { getProperties().const_offsets = ::mlir::Builder((*this)->getContext()).getDenseI64ArrayAttr(attrValue); } void StoreNdOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void StoreNdOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void StoreNdOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value TensorDesc, ArrayRef offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value TensorDesc, ArrayRef offsets, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = const_offsets; if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::mlir::DenseI64ArrayAttr const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreNdOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(TensorDesc); odsState.addOperands(offsets); odsState.getOrAddProperties().const_offsets = odsBuilder.getDenseI64ArrayAttr(const_offsets); if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value TensorDesc, ::mlir::ValueRange offsets, ::llvm::ArrayRef const_offsets, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(TensorDesc), std::forward(offsets), std::forward(const_offsets), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void StoreNdOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } StoreNdOp StoreNdOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreNdOp StoreNdOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult StoreNdOp::verifyInvariantsImpl() { auto tblgen_const_offsets = getProperties().const_offsets; (void)tblgen_const_offsets; if (!tblgen_const_offsets) return emitOpError("requires attribute 'const_offsets'"); auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU4(*this, tblgen_const_offsets, "const_offsets"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {value, TensorDesc} have same element type"); return ::mlir::success(); } ::llvm::LogicalResult StoreNdOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult StoreNdOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand TensorDescRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> TensorDescOperands(&TensorDescRawOperand, 1); ::llvm::SMLoc TensorDescOperandsLoc; (void)TensorDescOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> offsetsOperands; ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::DenseI64ArrayAttr const_offsetsAttr; ::mlir::Type valueRawType{}; ::llvm::ArrayRef<::mlir::Type> valueTypes(&valueRawType, 1); ::mlir::Type TensorDescRawType{}; ::llvm::ArrayRef<::mlir::Type> TensorDescTypes(&TensorDescRawType, 1); valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); TensorDescOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(TensorDescRawOperand)) return ::mlir::failure(); { offsetsOperandsLoc = parser.getCurrentLocation(); auto odsResult = parseDynamicIndexList(parser, offsetsOperands, const_offsetsAttr); if (odsResult) return ::mlir::failure(); result.getOrAddProperties().const_offsets = const_offsetsAttr; } if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueRawType = type; } if (parser.parseComma()) return ::mlir::failure(); if (parser.parseType(TensorDescRawType)) return ::mlir::failure(); ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(valueOperands, valueTypes, valueOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(TensorDescOperands, TensorDescTypes, TensorDescOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(offsetsOperands, odsBuildableType0, offsetsOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult StoreNdOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void StoreNdOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTensorDesc(); printDynamicIndexList(_odsPrinter, *this, getOffsets(), getConstOffsetsAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; elidedProps.push_back("const_offsets"); _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("const_offsets"); _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getValue().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTensorDesc().getType(); } void StoreNdOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreNdOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreNdOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::StoreScatterOp definitions //===----------------------------------------------------------------------===// namespace detail { StoreScatterOpGenericAdaptorBase::StoreScatterOpGenericAdaptorBase(StoreScatterOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::std::optional StoreScatterOpGenericAdaptorBase::getChunkSize() { auto attr = getChunkSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOpGenericAdaptorBase::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreScatterOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } } // namespace detail StoreScatterOpAdaptor::StoreScatterOpAdaptor(StoreScatterOp op) : StoreScatterOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult StoreScatterOpAdaptor::verify(::mlir::Location loc) { auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size; auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_chunk_size && !(((::llvm::isa<::mlir::IntegerAttr>(tblgen_chunk_size))) && ((::llvm::cast<::mlir::IntegerAttr>(tblgen_chunk_size).getType().isSignlessInteger(64))))) return emitError(loc, "'xegpu.store' op attribute 'chunk_size' failed to satisfy constraint: 64-bit signless integer attribute"); if (tblgen_l1_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l1_hint)))) return emitError(loc, "'xegpu.store' op attribute 'l1_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l2_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l2_hint)))) return emitError(loc, "'xegpu.store' op attribute 'l2_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_l3_hint && !((::llvm::isa<::mlir::xegpu::CachePolicyAttr>(tblgen_l3_hint)))) return emitError(loc, "'xegpu.store' op attribute 'l3_hint' failed to satisfy constraint: Describe the cache settings for prefetch/load/store operators"); if (tblgen_layout && !((::llvm::isa<::mlir::xegpu::DistributeLayoutAttr>(tblgen_layout)))) return emitError(loc, "'xegpu.store' op attribute 'layout' failed to satisfy constraint: DistributeLayoutAttr instance"); return ::mlir::success(); } ::llvm::LogicalResult StoreScatterOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.chunk_size; auto attr = dict.get("chunk_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto attr = dict.get("l1_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto attr = dict.get("l2_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto attr = dict.get("l3_hint"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute StoreScatterOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.chunk_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("chunk_size", propStorage)); } { const auto &propStorage = prop.l1_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l1_hint", propStorage)); } { const auto &propStorage = prop.l2_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l2_hint", propStorage)); } { const auto &propStorage = prop.l3_hint; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("l3_hint", propStorage)); } { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code StoreScatterOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.chunk_size.getAsOpaquePointer()), llvm::hash_value(prop.l1_hint.getAsOpaquePointer()), llvm::hash_value(prop.l2_hint.getAsOpaquePointer()), llvm::hash_value(prop.l3_hint.getAsOpaquePointer()), llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional StoreScatterOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "chunk_size") return prop.chunk_size; if (name == "l1_hint") return prop.l1_hint; if (name == "l2_hint") return prop.l2_hint; if (name == "l3_hint") return prop.l3_hint; if (name == "layout") return prop.layout; return std::nullopt; } void StoreScatterOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "chunk_size") { prop.chunk_size = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l1_hint") { prop.l1_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l2_hint") { prop.l2_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "l3_hint") { prop.l3_hint = ::llvm::dyn_cast_or_null>(value); return; } if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void StoreScatterOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.chunk_size) attrs.append("chunk_size", prop.chunk_size); if (prop.l1_hint) attrs.append("l1_hint", prop.l1_hint); if (prop.l2_hint) attrs.append("l2_hint", prop.l2_hint); if (prop.l3_hint) attrs.append("l3_hint", prop.l3_hint); if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult StoreScatterOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getChunkSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(attr, "chunk_size", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL1HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l1_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL2HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l2_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getL3HintAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(attr, "l3_hint", emitError))) return ::mlir::failure(); } { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult StoreScatterOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.chunk_size))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l1_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l2_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.l3_hint))) return ::mlir::failure(); if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void StoreScatterOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.chunk_size); writer.writeOptionalAttribute(prop.l1_hint); writer.writeOptionalAttribute(prop.l2_hint); writer.writeOptionalAttribute(prop.l3_hint); writer.writeOptionalAttribute(prop.layout); } ::std::optional StoreScatterOp::getChunkSize() { auto attr = getChunkSizeAttr(); return attr ? ::std::optional(attr.getValue().getZExtValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL1Hint() { auto attr = getL1HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL2Hint() { auto attr = getL2HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::CachePolicy> StoreScatterOp::getL3Hint() { auto attr = getL3HintAttr(); return attr ? ::std::optional<::mlir::xegpu::CachePolicy>(attr.getValue()) : (::std::nullopt); } ::std::optional<::mlir::xegpu::DistributeLayoutAttr> StoreScatterOp::getLayout() { auto attr = getLayoutAttr(); return attr ? ::std::optional<::mlir::xegpu::DistributeLayoutAttr>(attr) : (::std::nullopt); } void StoreScatterOp::setChunkSize(::std::optional attrValue) { auto &odsProp = getProperties().chunk_size; if (attrValue) odsProp = ::mlir::Builder((*this)->getContext()).getIntegerAttr(::mlir::Builder((*this)->getContext()).getIntegerType(64), *attrValue); else odsProp = nullptr; } void StoreScatterOp::setL1Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l1_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void StoreScatterOp::setL2Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l2_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } void StoreScatterOp::setL3Hint(::std::optional<::mlir::xegpu::CachePolicy> attrValue) { auto &odsProp = getProperties().l3_hint; if (attrValue) odsProp = ::mlir::xegpu::CachePolicyAttr::get(::mlir::Builder((*this)->getContext()).getContext(), *attrValue); else odsProp = nullptr; } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value dest, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value dest, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint) { return create(builder, builder.getLoc(), std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint)); } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value value, Value dest, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, Value value, Value dest, ArrayRef offsets, Value mask, IntegerAttr chunk_size, xegpu::CachePolicyAttr l1_hint, xegpu::CachePolicyAttr l2_hint, xegpu::CachePolicyAttr l3_hint, xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreScatterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(dest); odsState.addOperands(offsets); odsState.addOperands(mask); if (chunk_size) { odsState.getOrAddProperties().chunk_size = chunk_size; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreScatterOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { odsState.addOperands(value); odsState.addOperands(dest); odsState.addOperands(offsets); odsState.addOperands(mask); if (chunk_size) { odsState.getOrAddProperties().chunk_size = chunk_size; } if (l1_hint) { odsState.getOrAddProperties().l1_hint = l1_hint; } if (l2_hint) { odsState.getOrAddProperties().l2_hint = l2_hint; } if (l3_hint) { odsState.getOrAddProperties().l3_hint = l3_hint; } if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value value, ::mlir::Value dest, ::mlir::Value offsets, ::mlir::Value mask, /*optional*/::mlir::IntegerAttr chunk_size, /*optional*/::mlir::xegpu::CachePolicyAttr l1_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l2_hint, /*optional*/::mlir::xegpu::CachePolicyAttr l3_hint, /*optional*/::mlir::xegpu::DistributeLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(value), std::forward(dest), std::forward(offsets), std::forward(mask), std::forward(chunk_size), std::forward(l1_hint), std::forward(l2_hint), std::forward(l3_hint), std::forward(layout)); } void StoreScatterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void StoreScatterOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } StoreScatterOp StoreScatterOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreScatterOp StoreScatterOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult StoreScatterOp::verifyInvariantsImpl() { auto tblgen_chunk_size = getProperties().chunk_size; (void)tblgen_chunk_size; auto tblgen_l1_hint = getProperties().l1_hint; (void)tblgen_l1_hint; auto tblgen_l2_hint = getProperties().l2_hint; (void)tblgen_l2_hint; auto tblgen_l3_hint = getProperties().l3_hint; (void)tblgen_l3_hint; auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU1(*this, tblgen_chunk_size, "chunk_size"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l1_hint, "l1_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l2_hint, "l2_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU7(*this, tblgen_l3_hint, "l3_hint"))) return ::mlir::failure(); if (::mlir::failed(__mlir_ods_local_attr_constraint_XeGPU3(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU16(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU13(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult StoreScatterOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult StoreScatterOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueOperands(&valueRawOperand, 1); ::llvm::SMLoc valueOperandsLoc; (void)valueOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand destRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> destOperands(&destRawOperand, 1); ::llvm::SMLoc destOperandsLoc; (void)destOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand offsetsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> offsetsOperands(&offsetsRawOperand, 1); ::llvm::SMLoc offsetsOperandsLoc; (void)offsetsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; valueOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); destOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(destRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); offsetsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(offsetsRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); maskOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(maskRawOperand)) return ::mlir::failure(); if (parseProperties(parser, result)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.resolveOperands(::llvm::concat(valueOperands, destOperands, offsetsOperands, maskOperands), allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } ::llvm::LogicalResult StoreScatterOp::setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } // keep track of used keys in the input dictionary to be able to error out // if there are some unknown ones. ::mlir::DenseSet<::mlir::StringAttr> usedKeys; ::mlir::MLIRContext *ctx = dict.getContext(); (void)ctx; { auto &propStorage = prop.chunk_size; auto chunk_sizeAttrName = ::mlir::StringAttr::get(ctx, "chunk_size"); auto attr = dict.get(chunk_sizeAttrName); usedKeys.insert(chunk_sizeAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for chunk_size in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `chunk_size` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l1_hint; auto l1_hintAttrName = ::mlir::StringAttr::get(ctx, "l1_hint"); auto attr = dict.get(l1_hintAttrName); usedKeys.insert(l1_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l1_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l1_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l2_hint; auto l2_hintAttrName = ::mlir::StringAttr::get(ctx, "l2_hint"); auto attr = dict.get(l2_hintAttrName); usedKeys.insert(l2_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l2_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l2_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.l3_hint; auto l3_hintAttrName = ::mlir::StringAttr::get(ctx, "l3_hint"); auto attr = dict.get(l3_hintAttrName); usedKeys.insert(l3_hintAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for l3_hint in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `l3_hint` in property conversion: " << attr; return ::mlir::failure(); } } } { auto &propStorage = prop.layout; auto layoutAttrName = ::mlir::StringAttr::get(ctx, "layout"); auto attr = dict.get(layoutAttrName); usedKeys.insert(layoutAttrName); if (attr || /*isRequired=*/false) { if (!attr) { emitError() << "expected key entry for layout in DictionaryAttr to set " "Properties."; return ::mlir::failure(); } auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } for (::mlir::NamedAttribute attr : dict) { if (!usedKeys.contains(attr.getName())) return emitError() << "unknown key '" << attr.getName() << "' when parsing properties dictionary"; } return ::mlir::success(); } void StoreScatterOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValue(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getDest(); _odsPrinter << "["; _odsPrinter << getOffsets(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMask(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps; _odsPrinter << " "; printProperties(this->getContext(), _odsPrinter, getProperties(), elidedProps); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict(llvm::to_vector((*this)->getDiscardableAttrs()), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); } void StoreScatterOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::StoreScatterOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::detail::StoreScatterOpGenericAdaptorBase::Properties) namespace mlir::xegpu { //===----------------------------------------------------------------------===// // ::mlir::xegpu::TruncfOp definitions //===----------------------------------------------------------------------===// namespace detail { TruncfOpGenericAdaptorBase::TruncfOpGenericAdaptorBase(TruncfOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail TruncfOpAdaptor::TruncfOpAdaptor(TruncfOp op) : TruncfOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TruncfOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult TruncfOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TruncfOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TruncfOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional TruncfOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void TruncfOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void TruncfOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult TruncfOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void TruncfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value source) { odsState.addOperands(source); odsState.addTypes(result); } TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value source) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(source)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value source) { return create(builder, builder.getLoc(), std::forward(result), std::forward(source)); } void TruncfOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value source) { odsState.addOperands(source); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value source) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(source)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value source) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(source)); } void TruncfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TruncfOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TruncfOp TruncfOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TruncfOp TruncfOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult TruncfOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU17(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_XeGPU17(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getRank()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getRank()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getRank()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getRank())))) return emitOpError("failed to verify that all of {source, result} have same rank"); if (!(((::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) && (::llvm::cast<::mlir::ShapedType>((*this->getODSResults(0).begin()).getType()).getShape()) == (::llvm::cast<::mlir::ShapedType>((*this->getODSOperands(0).begin()).getType()).getShape())))) return emitOpError("failed to verify that all of {source, result} have same shape"); return ::mlir::success(); } ::llvm::LogicalResult TruncfOp::verifyInvariants() { if(::mlir::succeeded(verifyInvariantsImpl()) && ::mlir::succeeded(verify())) return ::mlir::success(); return ::mlir::failure(); } ::mlir::ParseResult TruncfOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand sourceRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> sourceOperands(&sourceRawOperand, 1); ::llvm::SMLoc sourceOperandsLoc; (void)sourceOperandsLoc; ::llvm::SmallVector<::mlir::Type, 1> allOperandTypes; ::llvm::SmallVector<::mlir::Type, 1> allResultTypes; sourceOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(sourceRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); if (parser.parseTypeList(allOperandTypes)) return ::mlir::failure(); if (parser.parseArrow()) return ::mlir::failure(); if (parser.parseTypeList(allResultTypes)) return ::mlir::failure(); result.addTypes(allResultTypes); if (parser.resolveOperands(sourceOperands, allOperandTypes, parser.getNameLoc(), result.operands)) return ::mlir::failure(); return ::mlir::success(); } void TruncfOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getSource(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; _odsPrinter << getOperation()->getOperandTypes(); _odsPrinter << ' ' << "->"; _odsPrinter << ' '; _odsPrinter << getOperation()->getResultTypes(); } void TruncfOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::xegpu MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::xegpu::TruncfOp) #endif // GET_OP_CLASSES