/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ |* *| |* Op Definitions *| |* *| |* Automatically generated file, do not edit! *| |* From: ArmSMEOps.td *| |* *| \*===----------------------------------------------------------------------===*/ #ifdef GET_OP_LIST #undef GET_OP_LIST ::mlir::arm_sme::CopyTileOp, ::mlir::arm_sme::ExtractTileSliceOp, ::mlir::arm_sme::FMopa2WayOp, ::mlir::arm_sme::FMops2WayOp, ::mlir::arm_sme::GetTileOp, ::mlir::arm_sme::InsertTileSliceOp, ::mlir::arm_sme::LoadTileSliceOp, ::mlir::arm_sme::OuterProductOp, ::mlir::arm_sme::SMopa2WayOp, ::mlir::arm_sme::SMopa4WayOp, ::mlir::arm_sme::SMops2WayOp, ::mlir::arm_sme::SMops4WayOp, ::mlir::arm_sme::StoreTileSliceOp, ::mlir::arm_sme::StreamingVLOp, ::mlir::arm_sme::SuMopa4WayOp, ::mlir::arm_sme::SuMops4WayOp, ::mlir::arm_sme::TileLoadOp, ::mlir::arm_sme::TileStoreOp, ::mlir::arm_sme::UMopa2WayOp, ::mlir::arm_sme::UMopa4WayOp, ::mlir::arm_sme::UMops2WayOp, ::mlir::arm_sme::UMops4WayOp, ::mlir::arm_sme::UsMopa4WayOp, ::mlir::arm_sme::UsMops4WayOp, ::mlir::arm_sme::ZeroOp #endif // GET_OP_LIST #ifdef GET_OP_CLASSES #undef GET_OP_CLASSES //===----------------------------------------------------------------------===// // Local Utility Method Definitions //===----------------------------------------------------------------------===// static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps1( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({16, 16})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({1, 1})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(128)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isF16()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::BFloat16Type>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isF64()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be a vector type that fits into a SME tile, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps2( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps3( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignlessInteger(128))) || ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((elementType.isF32())) || ((elementType.isF64())); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && (((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 1)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be a vector type that matches the size of a SVE vector, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps4( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be of ranks 1scalable vector of 16-bit float or bfloat16 type values of length 8, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps5( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps6( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector<[4]x[4]xf32> of 32-bit float values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps7( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps8( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(1)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && (((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 1)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be a vector type that matches the size of a SVE predicate, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps9( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((::llvm::isa<::mlir::IndexType>(type)))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be variadic of index, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps10( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps11( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector<[4]x[4]xi32> of 32-bit signless integer values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps12( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps13( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps14( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!((((::llvm::isa<::mlir::MemRefType>(type))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::cast<::mlir::ShapedType>(type).hasRank())) && ((::llvm::cast<::mlir::ShapedType>(type).getRank() == 2))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be 2D memref of any non-token type values, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_type_constraint_ArmSMEOps15( ::mlir::Operation *op, ::mlir::Type type, ::llvm::StringRef valueKind, unsigned valueIndex) { if (!(!((::llvm::isa<::mlir::TokenType>(type))))) { return op->emitOpError(valueKind) << " #" << valueIndex << " must be any non-token type, but got " << type; } return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps1( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Layout of a tile slice"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps1( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_ArmSMEOps1(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps2( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::arm_sme::CombiningKindAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Kind of combining function"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps2( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_ArmSMEOps2(attr, attrName, [op]() { return op->emitOpError(); }); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps3( ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { if (attr && !((::llvm::isa<::mlir::arm_sme::TypeSizeAttr>(attr)))) return emitError() << "attribute '" << attrName << "' failed to satisfy constraint: Size of a vector element type"; return ::mlir::success(); } static ::llvm::LogicalResult __mlir_ods_local_attr_constraint_ArmSMEOps3( ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) { return __mlir_ods_local_attr_constraint_ArmSMEOps3(attr, attrName, [op]() { return op->emitOpError(); }); } namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::CopyTileOp definitions //===----------------------------------------------------------------------===// namespace detail { CopyTileOpGenericAdaptorBase::CopyTileOpGenericAdaptorBase(CopyTileOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail CopyTileOpAdaptor::CopyTileOpAdaptor(CopyTileOp op) : CopyTileOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult CopyTileOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult CopyTileOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute CopyTileOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code CopyTileOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional CopyTileOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void CopyTileOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void CopyTileOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult CopyTileOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void CopyTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value tile) { odsState.addOperands(tile); odsState.addTypes(result); } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value tile) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(tile)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value tile) { return create(builder, builder.getLoc(), std::forward(result), std::forward(tile)); } void CopyTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tile) { odsState.addOperands(tile); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(CopyTileOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tile) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tile) { return create(builder, builder.getLoc(), std::forward(tile)); } void CopyTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tile) { odsState.addOperands(tile); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tile) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tile)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tile) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tile)); } void CopyTileOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void CopyTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(CopyTileOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void CopyTileOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void CopyTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(CopyTileOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } CopyTileOp CopyTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } CopyTileOp CopyTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult CopyTileOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {tile, result} have same type"); return ::mlir::success(); } ::llvm::LogicalResult CopyTileOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult CopyTileOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[0].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult CopyTileOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tileRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tileOperands(&tileRawOperand, 1); ::llvm::SMLoc tileOperandsLoc; (void)tileOperandsLoc; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); tileOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tileRawOperand)) return ::mlir::failure(); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } result.addTypes(resultTypes); if (parser.resolveOperands(tileOperands, resultTypes[0], tileOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void CopyTileOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTile(); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void CopyTileOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::CopyTileOp) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::ExtractTileSliceOp definitions //===----------------------------------------------------------------------===// namespace detail { ExtractTileSliceOpGenericAdaptorBase::ExtractTileSliceOpGenericAdaptorBase(ExtractTileSliceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::arm_sme::TileSliceLayoutAttr ExtractTileSliceOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout ExtractTileSliceOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail ExtractTileSliceOpAdaptor::ExtractTileSliceOpAdaptor(ExtractTileSliceOp op) : ExtractTileSliceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ExtractTileSliceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.extract_tile_slice' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } ::llvm::LogicalResult ExtractTileSliceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute ExtractTileSliceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ExtractTileSliceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional ExtractTileSliceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; return std::nullopt; } void ExtractTileSliceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void ExtractTileSliceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult ExtractTileSliceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult ExtractTileSliceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void ExtractTileSliceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); } ::mlir::arm_sme::TileSliceLayout ExtractTileSliceOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void ExtractTileSliceOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ExtractTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); odsState.addTypes(result); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ExtractTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ExtractTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ExtractTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(ExtractTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ExtractTileSliceOp ExtractTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void ExtractTileSliceOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult ExtractTileSliceOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps3(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((std::equal_to<>()(VectorType(VectorType::Builder(::llvm::cast((*this->getODSOperands(0).begin()).getType())).dropDim(0)), (*this->getODSResults(0).begin()).getType())))) return emitOpError("failed to verify that type of 'result' matches type of 'tile' slice"); return ::mlir::success(); } ::llvm::LogicalResult ExtractTileSliceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult ExtractTileSliceOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = VectorType(VectorType::Builder(::llvm::cast(operands[0].getType())).dropDim(0)); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult ExtractTileSliceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tileRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tileOperands(&tileRawOperand, 1); ::llvm::SMLoc tileOperandsLoc; (void)tileOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tile_slice_indexRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tile_slice_indexOperands(&tile_slice_indexRawOperand, 1); ::llvm::SMLoc tile_slice_indexOperandsLoc; (void)tile_slice_indexOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); ::mlir::Type tileRawType{}; ::llvm::ArrayRef<::mlir::Type> tileTypes(&tileRawType, 1); tileOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tileRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); tile_slice_indexOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tile_slice_indexRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } if (parser.parseKeyword("from")) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tileRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resultTypes); if (parser.resolveOperands(tileOperands, tileTypes, tileOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tile_slice_indexOperands, odsBuildableType0, tile_slice_indexOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void ExtractTileSliceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTile(); _odsPrinter << "["; _odsPrinter << getTileSliceIndex(); _odsPrinter << "]"; if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "from"; _odsPrinter << ' '; { auto type = getTile().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void ExtractTileSliceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::ExtractTileSliceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::ExtractTileSliceOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::FMopa2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { FMopa2WayOpGenericAdaptorBase::FMopa2WayOpGenericAdaptorBase(FMopa2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair FMopa2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail FMopa2WayOpAdaptor::FMopa2WayOpAdaptor(FMopa2WayOp op) : FMopa2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult FMopa2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair FMopa2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange FMopa2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange FMopa2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange FMopa2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult FMopa2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute FMopa2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code FMopa2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional FMopa2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void FMopa2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void FMopa2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult FMopa2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult FMopa2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void FMopa2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void FMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } FMopa2WayOp FMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMopa2WayOp FMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void FMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } FMopa2WayOp FMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMopa2WayOp FMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void FMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } FMopa2WayOp FMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMopa2WayOp FMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void FMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } FMopa2WayOp FMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMopa2WayOp FMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult FMopa2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult FMopa2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult FMopa2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit float or bfloat16 type values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xf32> of 32-bit float values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void FMopa2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void FMopa2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::FMopa2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::FMopa2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::FMops2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { FMops2WayOpGenericAdaptorBase::FMops2WayOpGenericAdaptorBase(FMops2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair FMops2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail FMops2WayOpAdaptor::FMops2WayOpAdaptor(FMops2WayOp op) : FMops2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult FMops2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair FMops2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange FMops2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange FMops2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange FMops2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult FMops2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute FMops2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code FMops2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional FMops2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void FMops2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void FMops2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult FMops2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult FMops2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void FMops2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void FMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } FMops2WayOp FMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMops2WayOp FMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void FMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } FMops2WayOp FMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMops2WayOp FMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void FMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } FMops2WayOp FMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMops2WayOp FMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void FMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } FMops2WayOp FMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } FMops2WayOp FMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult FMops2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps4(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps6(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult FMops2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult FMops2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit float or bfloat16 type values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xf32> of 32-bit float values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void FMops2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void FMops2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::FMops2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::FMops2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::GetTileOp definitions //===----------------------------------------------------------------------===// namespace detail { GetTileOpGenericAdaptorBase::GetTileOpGenericAdaptorBase(GetTileOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail GetTileOpAdaptor::GetTileOpAdaptor(GetTileOp op) : GetTileOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult GetTileOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult GetTileOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute GetTileOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code GetTileOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional GetTileOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void GetTileOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void GetTileOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult GetTileOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void GetTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type tile) { odsState.addTypes(tile); } GetTileOp GetTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type tile) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GetTileOp GetTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type tile) { return create(builder, builder.getLoc(), std::forward(tile)); } void GetTileOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } GetTileOp GetTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GetTileOp GetTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void GetTileOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } GetTileOp GetTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GetTileOp GetTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void GetTileOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } GetTileOp GetTileOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } GetTileOp GetTileOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult GetTileOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult GetTileOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult GetTileOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::Type tileRawType{}; ::llvm::ArrayRef<::mlir::Type> tileTypes(&tileRawType, 1); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tileRawType = type; } result.addTypes(tileTypes); return ::mlir::success(); } void GetTileOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getTile().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void GetTileOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::GetTileOp) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::InsertTileSliceOp definitions //===----------------------------------------------------------------------===// namespace detail { InsertTileSliceOpGenericAdaptorBase::InsertTileSliceOpGenericAdaptorBase(InsertTileSliceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::arm_sme::TileSliceLayoutAttr InsertTileSliceOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout InsertTileSliceOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail InsertTileSliceOpAdaptor::InsertTileSliceOpAdaptor(InsertTileSliceOp op) : InsertTileSliceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult InsertTileSliceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.insert_tile_slice' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } ::llvm::LogicalResult InsertTileSliceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute InsertTileSliceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code InsertTileSliceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional InsertTileSliceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; return std::nullopt; } void InsertTileSliceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void InsertTileSliceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult InsertTileSliceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult InsertTileSliceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void InsertTileSliceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); } ::mlir::arm_sme::TileSliceLayout InsertTileSliceOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void InsertTileSliceOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InsertTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); odsState.addTypes(result); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InsertTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(vector); odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value vector, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(vector), std::forward(tile), std::forward(tile_slice_index), std::forward(layout)); } void InsertTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InsertTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void InsertTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void InsertTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 3u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(InsertTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } InsertTileSliceOp InsertTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } InsertTileSliceOp InsertTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void InsertTileSliceOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult InsertTileSliceOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType())))) return emitOpError("failed to verify that all of {tile, result} have same type"); if (!((std::equal_to<>()(VectorType::get(::llvm::cast((*this->getODSOperands(1).begin()).getType()).getShape().drop_front(),::llvm::cast((*this->getODSOperands(1).begin()).getType()).getElementType(),/*scalableDims=*/{true}), (*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that type of 'vector' matches type of 'tile' slice"); return ::mlir::success(); } ::llvm::LogicalResult InsertTileSliceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult InsertTileSliceOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 1) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[1].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult InsertTileSliceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand vectorRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> vectorOperands(&vectorRawOperand, 1); ::llvm::SMLoc vectorOperandsLoc; (void)vectorOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tileRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tileOperands(&tileRawOperand, 1); ::llvm::SMLoc tileOperandsLoc; (void)tileOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tile_slice_indexRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tile_slice_indexOperands(&tile_slice_indexRawOperand, 1); ::llvm::SMLoc tile_slice_indexOperandsLoc; (void)tile_slice_indexOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type vectorRawType{}; ::llvm::ArrayRef<::mlir::Type> vectorTypes(&vectorRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); vectorOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(vectorRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tileOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tileRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); tile_slice_indexOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tile_slice_indexRawOperand)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); vectorRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resultTypes); if (parser.resolveOperands(vectorOperands, vectorTypes, vectorOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tileOperands, resultTypes[0], tileOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tile_slice_indexOperands, odsBuildableType0, tile_slice_indexOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void InsertTileSliceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getVector(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTile(); _odsPrinter << "["; _odsPrinter << getTileSliceIndex(); _odsPrinter << "]"; if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getVector().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void InsertTileSliceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::InsertTileSliceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::InsertTileSliceOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::LoadTileSliceOp definitions //===----------------------------------------------------------------------===// namespace detail { LoadTileSliceOpGenericAdaptorBase::LoadTileSliceOpGenericAdaptorBase(LoadTileSliceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair LoadTileSliceOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, false, true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::arm_sme::TileSliceLayoutAttr LoadTileSliceOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout LoadTileSliceOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail LoadTileSliceOpAdaptor::LoadTileSliceOpAdaptor(LoadTileSliceOp op) : LoadTileSliceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult LoadTileSliceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.load_tile_slice' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } std::pair LoadTileSliceOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, false, true, false}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange LoadTileSliceOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult LoadTileSliceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute LoadTileSliceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code LoadTileSliceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional LoadTileSliceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; return std::nullopt; } void LoadTileSliceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void LoadTileSliceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult LoadTileSliceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult LoadTileSliceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void LoadTileSliceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); } ::mlir::arm_sme::TileSliceLayout LoadTileSliceOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void LoadTileSliceOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LoadTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); odsState.addTypes(result); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LoadTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(base); odsState.addOperands(mask); odsState.addOperands(tile); odsState.addOperands(indices); odsState.addOperands(tile_slice_index); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::Value mask, ::mlir::Value tile, ::mlir::ValueRange indices, ::mlir::Value tile_slice_index, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(base), std::forward(mask), std::forward(tile), std::forward(indices), std::forward(tile_slice_index), std::forward(layout)); } void LoadTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LoadTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void LoadTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void LoadTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(LoadTileSliceOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } LoadTileSliceOp LoadTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } LoadTileSliceOp LoadTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void LoadTileSliceOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult LoadTileSliceOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(2).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(2).begin())))))) return emitOpError("failed to verify that all of {tile, base} have same element type"); if (!((((*this->getODSOperands(2).begin()).getType()) == ((*this->getODSResults(0).begin()).getType()) && ((*this->getODSResults(0).begin()).getType()) == ((*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that all of {tile, result} have same type"); if (!((std::equal_to<>()(VectorType(VectorType::Builder(::llvm::cast((*this->getODSResults(0).begin()).getType())).dropDim(0).setElementType(IntegerType::get((*this->getODSResults(0).begin()).getType().getContext(), 1))), (*this->getODSOperands(1).begin()).getType())))) return emitOpError("failed to verify that `mask` has i1 element type and the shape is a slice of `result`"); return ::mlir::success(); } ::llvm::LogicalResult LoadTileSliceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult LoadTileSliceOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 2) return ::mlir::failure(); ::mlir::Type odsInferredType0 = operands[2].getType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult LoadTileSliceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand baseRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> baseOperands(&baseRawOperand, 1); ::llvm::SMLoc baseOperandsLoc; (void)baseOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tileRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tileOperands(&tileRawOperand, 1); ::llvm::SMLoc tileOperandsLoc; (void)tileOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tile_slice_indexRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tile_slice_indexOperands(&tile_slice_indexRawOperand, 1); ::llvm::SMLoc tile_slice_indexOperandsLoc; (void)tile_slice_indexOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type baseRawType{}; ::llvm::ArrayRef<::mlir::Type> baseTypes(&baseRawType, 1); ::mlir::Type maskRawType{}; ::llvm::ArrayRef<::mlir::Type> maskTypes(&maskRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); baseOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(baseRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); maskOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(maskRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tileOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tileRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tile_slice_indexOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tile_slice_indexRawOperand)) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); baseRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); maskRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resultTypes); if (parser.resolveOperands(baseOperands, baseTypes, baseOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(maskOperands, maskTypes, maskOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tileOperands, resultTypes[0], tileOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tile_slice_indexOperands, odsBuildableType0, tile_slice_indexOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void LoadTileSliceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBase(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMask(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTile(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTileSliceIndex(); if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBase().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMask().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void LoadTileSliceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::LoadTileSliceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::LoadTileSliceOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::OuterProductOp definitions //===----------------------------------------------------------------------===// namespace detail { OuterProductOpGenericAdaptorBase::OuterProductOpGenericAdaptorBase(OuterProductOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair OuterProductOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::arm_sme::CombiningKindAttr OuterProductOpGenericAdaptorBase::getKindAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::CombiningKindAttr>(getProperties().kind); return attr; } ::mlir::arm_sme::CombiningKind OuterProductOpGenericAdaptorBase::getKind() { auto attr = getKindAttr(); return attr.getValue(); } } // namespace detail OuterProductOpAdaptor::OuterProductOpAdaptor(OuterProductOp op) : OuterProductOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult OuterProductOpAdaptor::verify(::mlir::Location loc) { auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (tblgen_kind && !((::llvm::isa<::mlir::arm_sme::CombiningKindAttr>(tblgen_kind)))) return emitError(loc, "'arm_sme.outerproduct' op attribute 'kind' failed to satisfy constraint: Kind of combining function"); return ::mlir::success(); } std::pair OuterProductOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange OuterProductOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange OuterProductOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange OuterProductOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult OuterProductOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.kind; auto attr = dict.get("kind"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `kind` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute OuterProductOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.kind; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("kind", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code OuterProductOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.kind.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional OuterProductOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "kind") return prop.kind; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void OuterProductOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "kind") { prop.kind = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void OuterProductOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.kind) attrs.append("kind", prop.kind); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult OuterProductOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getKindAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps2(attr, "kind", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult OuterProductOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.kind))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void OuterProductOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.kind); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::arm_sme::CombiningKind OuterProductOp::getKind() { auto attr = getKindAttr(); return attr.getValue(); } void OuterProductOp::setKind(::mlir::arm_sme::CombiningKind attrValue) { getProperties().kind = ::mlir::arm_sme::CombiningKindAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (kind) { odsState.getOrAddProperties().kind = kind; } odsState.addTypes(result); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (kind) { odsState.getOrAddProperties().kind = kind; } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(OuterProductOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { return create(builder, builder.getLoc(), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (kind) { odsState.getOrAddProperties().kind = kind; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKindAttr kind) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().kind = ::mlir::arm_sme::CombiningKindAttr::get(odsBuilder.getContext(), kind); odsState.addTypes(result); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().kind = ::mlir::arm_sme::CombiningKindAttr::get(odsBuilder.getContext(), kind); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(OuterProductOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { return create(builder, builder.getLoc(), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().kind = ::mlir::arm_sme::CombiningKindAttr::get(odsBuilder.getContext(), kind); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc, ::mlir::arm_sme::CombiningKind kind) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc), std::forward(kind)); } void OuterProductOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(OuterProductOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void OuterProductOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void OuterProductOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(OuterProductOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } OuterProductOp OuterProductOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } OuterProductOp OuterProductOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void OuterProductOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.kind) properties.kind = ::mlir::arm_sme::CombiningKindAttr::get(odsBuilder.getContext(), CombiningKind::Add); } ::llvm::LogicalResult OuterProductOp::verifyInvariantsImpl() { auto tblgen_kind = getProperties().kind; (void)tblgen_kind; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps2(*this, tblgen_kind, "kind"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps3(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getLhs() || !getResult() || std::equal_to<>()([&]{ auto vectorType = ::llvm::cast((*this->getODSOperands(0).begin()).getType()); int64_t size = vectorType.getDimSize(0); return VectorType::get( { size, size }, vectorType.getElementType(), { true, true });}(), (*this->getODSResults(0).begin()).getType())))) return emitOpError("failed to verify that resulttype is derived from `lhs` and `rhs`"); if (!((!getLhs() || !getAcc() || std::equal_to<>()([&]{ auto vectorType = ::llvm::cast((*this->getODSOperands(0).begin()).getType()); int64_t size = vectorType.getDimSize(0); return VectorType::get( { size, size }, vectorType.getElementType(), { true, true });}(), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that acctype is derived from `lhs` and `rhs`"); return ::mlir::success(); } ::llvm::LogicalResult OuterProductOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult OuterProductOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); if (operands.size() <= 0) return ::mlir::failure(); ::mlir::Type odsInferredType0 = [&]{ auto vectorType = ::llvm::cast(operands[0].getType()); int64_t size = vectorType.getDimSize(0); return VectorType::get( { size, size }, vectorType.getElementType(), { true, true });}(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult OuterProductOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::mlir::arm_sme::CombiningKindAttr kindAttr; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool kindClause = false; bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("kind"))) { if (kindClause) { return parser.emitError(parser.getNameLoc()) << "`kind` clause can appear at most once in the expansion of the " "oilist directive"; } kindClause = true; if (parser.parseCustomAttributeWithFallback(kindAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (kindAttr) result.getOrAddProperties().kind = kindAttr; } else if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignlessInteger(128))) || ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((elementType.isF32())) || ((elementType.isF64())); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && (((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 1)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be a vector type that matches the size of a SVE vector, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return ((elementType.isSignlessInteger(8))) || ((elementType.isSignlessInteger(16))) || ((elementType.isSignlessInteger(32))) || ((elementType.isSignlessInteger(64))) || ((elementType.isSignlessInteger(128))) || ((elementType.isF16())) || ((::llvm::isa<::mlir::BFloat16Type>(elementType))) || ((elementType.isF32())) || ((elementType.isF64())); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && (((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 4)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 2)) || ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 1)))))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be a vector type that matches the size of a SVE vector, but got " << type; } } result.addTypes([&]{ auto vectorType = ::llvm::cast(lhsTypes[0]); int64_t size = vectorType.getDimSize(0); return VectorType::get( { size, size }, vectorType.getElementType(), { true, true });}()); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, [&]{ auto vectorType = ::llvm::cast(lhsTypes[0]); int64_t size = vectorType.getDimSize(0); return VectorType::get( { size, size }, vectorType.getElementType(), { true, true });}(), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void OuterProductOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || (getKindAttr() != ::mlir::arm_sme::CombiningKindAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), CombiningKind::Add))) { _odsPrinter << ' ' << "kind"; _odsPrinter.printStrippedAttrOrType(getKindAttr()); } if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("kind"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getKindAttr(); if(attr && (attr == ::mlir::arm_sme::CombiningKindAttr::get(odsBuilder.getContext(), CombiningKind::Add))) elidedAttrs.push_back("kind"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void OuterProductOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::OuterProductOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::OuterProductOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SMopa2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SMopa2WayOpGenericAdaptorBase::SMopa2WayOpGenericAdaptorBase(SMopa2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SMopa2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SMopa2WayOpAdaptor::SMopa2WayOpAdaptor(SMopa2WayOp op) : SMopa2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SMopa2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SMopa2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SMopa2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMopa2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMopa2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SMopa2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SMopa2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SMopa2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SMopa2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SMopa2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SMopa2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SMopa2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SMopa2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SMopa2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SMopa2WayOp SMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa2WayOp SMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SMopa2WayOp SMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa2WayOp SMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SMopa2WayOp SMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa2WayOp SMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SMopa2WayOp SMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa2WayOp SMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SMopa2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps11(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult SMopa2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SMopa2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SMopa2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SMopa2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SMopa2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SMopa2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SMopa4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SMopa4WayOpGenericAdaptorBase::SMopa4WayOpGenericAdaptorBase(SMopa4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SMopa4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SMopa4WayOpAdaptor::SMopa4WayOpAdaptor(SMopa4WayOp op) : SMopa4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SMopa4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SMopa4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SMopa4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMopa4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMopa4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SMopa4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SMopa4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SMopa4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SMopa4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SMopa4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SMopa4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SMopa4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SMopa4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SMopa4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SMopa4WayOp SMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa4WayOp SMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SMopa4WayOp SMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa4WayOp SMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SMopa4WayOp SMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa4WayOp SMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SMopa4WayOp SMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMopa4WayOp SMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SMopa4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult SMopa4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SMopa4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SMopa4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SMopa4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SMopa4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SMopa4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SMops2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SMops2WayOpGenericAdaptorBase::SMops2WayOpGenericAdaptorBase(SMops2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SMops2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SMops2WayOpAdaptor::SMops2WayOpAdaptor(SMops2WayOp op) : SMops2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SMops2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SMops2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SMops2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMops2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMops2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SMops2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SMops2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SMops2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SMops2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SMops2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SMops2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SMops2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SMops2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SMops2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SMops2WayOp SMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops2WayOp SMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SMops2WayOp SMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops2WayOp SMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SMops2WayOp SMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops2WayOp SMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SMops2WayOp SMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops2WayOp SMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SMops2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps11(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult SMops2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SMops2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SMops2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SMops2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SMops2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SMops2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SMops4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SMops4WayOpGenericAdaptorBase::SMops4WayOpGenericAdaptorBase(SMops4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SMops4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SMops4WayOpAdaptor::SMops4WayOpAdaptor(SMops4WayOp op) : SMops4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SMops4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SMops4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SMops4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMops4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SMops4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SMops4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SMops4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SMops4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SMops4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SMops4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SMops4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SMops4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SMops4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SMops4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SMops4WayOp SMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops4WayOp SMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SMops4WayOp SMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops4WayOp SMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SMops4WayOp SMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops4WayOp SMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SMops4WayOp SMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SMops4WayOp SMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SMops4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult SMops4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SMops4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SMops4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SMops4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SMops4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SMops4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::StoreTileSliceOp definitions //===----------------------------------------------------------------------===// namespace detail { StoreTileSliceOpGenericAdaptorBase::StoreTileSliceOpGenericAdaptorBase(StoreTileSliceOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair StoreTileSliceOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { bool isVariadic[] = {false, false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (odsOperandsSize - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::arm_sme::TileSliceLayoutAttr StoreTileSliceOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout StoreTileSliceOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail StoreTileSliceOpAdaptor::StoreTileSliceOpAdaptor(StoreTileSliceOp op) : StoreTileSliceOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult StoreTileSliceOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.store_tile_slice' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } std::pair StoreTileSliceOp::getODSOperandIndexAndLength(unsigned index) { bool isVariadic[] = {false, false, false, false, true}; int prevVariadicCount = 0; for (unsigned i = 0; i < index; ++i) if (isVariadic[i]) ++prevVariadicCount; // Calculate how many dynamic values a static variadic operand corresponds to. // This assumes all static variadic operands have the same dynamic value count. int variadicSize = (getOperation()->getNumOperands() - 4) / 1; // `index` passed in as the parameter is the static index which counts each // operand (variadic or not) as size 1. So here for each previous static variadic // operand, we need to offset by (variadicSize - 1) to get where the dynamic // value pack for this static operand starts. int start = index + (variadicSize - 1) * prevVariadicCount; int size = isVariadic[index] ? variadicSize : 1; return {start, size}; } ::mlir::MutableOperandRange StoreTileSliceOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second); return mutableRange; } ::llvm::LogicalResult StoreTileSliceOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute StoreTileSliceOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code StoreTileSliceOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer())); } std::optional StoreTileSliceOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; return std::nullopt; } void StoreTileSliceOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } } void StoreTileSliceOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); } ::llvm::LogicalResult StoreTileSliceOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult StoreTileSliceOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); return ::mlir::success(); } void StoreTileSliceOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); } ::mlir::arm_sme::TileSliceLayout StoreTileSliceOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void StoreTileSliceOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void StoreTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.addOperands(mask); odsState.addOperands(base); odsState.addOperands(indices); if (layout) { odsState.getOrAddProperties().layout = layout; } } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); } void StoreTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.addOperands(mask); odsState.addOperands(base); odsState.addOperands(indices); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); } void StoreTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.addOperands(mask); odsState.addOperands(base); odsState.addOperands(indices); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); } void StoreTileSliceOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(tile); odsState.addOperands(tile_slice_index); odsState.addOperands(mask); odsState.addOperands(base); odsState.addOperands(indices); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value tile, ::mlir::Value tile_slice_index, ::mlir::Value mask, ::mlir::Value base, ::mlir::ValueRange indices, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(tile), std::forward(tile_slice_index), std::forward(mask), std::forward(base), std::forward(indices), std::forward(layout)); } void StoreTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void StoreTileSliceOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 4u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } StoreTileSliceOp StoreTileSliceOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StoreTileSliceOp StoreTileSliceOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void StoreTileSliceOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult StoreTileSliceOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps2(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps8(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps7(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(3).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(3).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {tile, base} have same element type"); if (!((std::equal_to<>()(VectorType(VectorType::Builder(::llvm::cast((*this->getODSOperands(0).begin()).getType())).dropDim(0).setElementType(IntegerType::get((*this->getODSOperands(0).begin()).getType().getContext(), 1))), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that `mask` has i1 element type and the shape is a slice of `tile`"); return ::mlir::success(); } ::llvm::LogicalResult StoreTileSliceOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult StoreTileSliceOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand tileRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tileOperands(&tileRawOperand, 1); ::llvm::SMLoc tileOperandsLoc; (void)tileOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand tile_slice_indexRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> tile_slice_indexOperands(&tile_slice_indexRawOperand, 1); ::llvm::SMLoc tile_slice_indexOperandsLoc; (void)tile_slice_indexOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand maskRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> maskOperands(&maskRawOperand, 1); ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand baseRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> baseOperands(&baseRawOperand, 1); ::llvm::SMLoc baseOperandsLoc; (void)baseOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type baseRawType{}; ::llvm::ArrayRef<::mlir::Type> baseTypes(&baseRawType, 1); ::mlir::Type maskRawType{}; ::llvm::ArrayRef<::mlir::Type> maskTypes(&maskRawType, 1); ::mlir::Type tileRawType{}; ::llvm::ArrayRef<::mlir::Type> tileTypes(&tileRawType, 1); tileOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tileRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); tile_slice_indexOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(tile_slice_indexRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); maskOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(maskRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); baseOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(baseRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); baseRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); maskRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); tileRawType = type; } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(tileOperands, tileTypes, tileOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(tile_slice_indexOperands, odsBuildableType0, tile_slice_indexOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(maskOperands, maskTypes, maskOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(baseOperands, baseTypes, baseOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); return ::mlir::success(); } void StoreTileSliceOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getTile(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getTileSliceIndex(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getMask(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBase(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBase().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getMask().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getTile().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void StoreTileSliceOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(3); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::StoreTileSliceOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::StoreTileSliceOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::StreamingVLOp definitions //===----------------------------------------------------------------------===// namespace detail { StreamingVLOpGenericAdaptorBase::StreamingVLOpGenericAdaptorBase(StreamingVLOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} ::mlir::arm_sme::TypeSize StreamingVLOpGenericAdaptorBase::getTypeSize() { auto attr = getTypeSizeAttr(); return attr.getValue(); } } // namespace detail StreamingVLOpAdaptor::StreamingVLOpAdaptor(StreamingVLOp op) : StreamingVLOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult StreamingVLOpAdaptor::verify(::mlir::Location loc) { auto tblgen_type_size = getProperties().type_size; (void)tblgen_type_size; if (!tblgen_type_size) return emitError(loc, "'arm_sme.streaming_vl' op requires attribute 'type_size'"); if (tblgen_type_size && !((::llvm::isa<::mlir::arm_sme::TypeSizeAttr>(tblgen_type_size)))) return emitError(loc, "'arm_sme.streaming_vl' op attribute 'type_size' failed to satisfy constraint: Size of a vector element type"); return ::mlir::success(); } ::llvm::LogicalResult StreamingVLOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.type_size; auto attr = dict.get("type_size"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `type_size` in property conversion: " << attr; return ::mlir::failure(); } } } return ::mlir::success(); } ::mlir::Attribute StreamingVLOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.type_size; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("type_size", propStorage)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code StreamingVLOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.type_size.getAsOpaquePointer())); } std::optional StreamingVLOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "type_size") return prop.type_size; return std::nullopt; } void StreamingVLOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "type_size") { prop.type_size = ::llvm::dyn_cast_or_null>(value); return; } } void StreamingVLOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.type_size) attrs.append("type_size", prop.type_size); } ::llvm::LogicalResult StreamingVLOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getTypeSizeAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps3(attr, "type_size", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult StreamingVLOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readAttribute(prop.type_size))) return ::mlir::failure(); return ::mlir::success(); } void StreamingVLOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeAttribute(prop.type_size); } ::mlir::arm_sme::TypeSize StreamingVLOp::getTypeSize() { auto attr = getTypeSizeAttr(); return attr.getValue(); } void StreamingVLOp::setTypeSize(::mlir::arm_sme::TypeSize attrValue) { getProperties().type_size = ::mlir::arm_sme::TypeSizeAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSizeAttr type_size) { odsState.getOrAddProperties().type_size = type_size; odsState.addTypes(resultType0); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSizeAttr type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSizeAttr type_size) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::arm_sme::TypeSizeAttr type_size) { odsState.getOrAddProperties().type_size = type_size; ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(StreamingVLOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::arm_sme::TypeSizeAttr type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::arm_sme::TypeSizeAttr type_size) { return create(builder, builder.getLoc(), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSizeAttr type_size) { odsState.getOrAddProperties().type_size = type_size; assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSizeAttr type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSizeAttr type_size) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSize type_size) { odsState.getOrAddProperties().type_size = ::mlir::arm_sme::TypeSizeAttr::get(odsBuilder.getContext(), type_size); odsState.addTypes(resultType0); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSize type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType0), std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type resultType0, ::mlir::arm_sme::TypeSize type_size) { return create(builder, builder.getLoc(), std::forward(resultType0), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::arm_sme::TypeSize type_size) { odsState.getOrAddProperties().type_size = ::mlir::arm_sme::TypeSizeAttr::get(odsBuilder.getContext(), type_size); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(StreamingVLOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, odsState.operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) odsState.addTypes(inferredReturnTypes); else ::mlir::detail::reportFatalInferReturnTypesError(odsState); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::arm_sme::TypeSize type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::arm_sme::TypeSize type_size) { return create(builder, builder.getLoc(), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSize type_size) { odsState.getOrAddProperties().type_size = ::mlir::arm_sme::TypeSizeAttr::get(odsBuilder.getContext(), type_size); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSize type_size) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(type_size)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::arm_sme::TypeSize type_size) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(type_size)); } void StreamingVLOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(StreamingVLOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(attributes)); } void StreamingVLOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void StreamingVLOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes; if (::mlir::succeeded(StreamingVLOp::inferReturnTypes(odsBuilder.getContext(), odsState.location, operands, odsState.attributes.getDictionary(odsState.getContext()), odsState.getRawProperties(), odsState.regions, inferredReturnTypes))) { assert(inferredReturnTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(inferredReturnTypes); } else { ::llvm::report_fatal_error("Failed to infer result type(s)."); } } StreamingVLOp StreamingVLOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } StreamingVLOp StreamingVLOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult StreamingVLOp::verifyInvariantsImpl() { auto tblgen_type_size = getProperties().type_size; (void)tblgen_type_size; if (!tblgen_type_size) return emitOpError("requires attribute 'type_size'"); if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps3(*this, tblgen_type_size, "type_size"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps2(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult StreamingVLOp::verifyInvariants() { return verifyInvariantsImpl(); } ::llvm::LogicalResult StreamingVLOp::inferReturnTypes(::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, ::mlir::PropertyRef properties, ::mlir::RegionRange regions, ::llvm::SmallVectorImpl<::mlir::Type>&inferredReturnTypes) { inferredReturnTypes.resize(1); ::mlir::Builder odsBuilder(context); ::mlir::Type odsInferredType0 = odsBuilder.getIndexType(); inferredReturnTypes[0] = odsInferredType0; return ::mlir::success(); } ::mlir::ParseResult StreamingVLOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::arm_sme::TypeSizeAttr type_sizeAttr; if (parser.parseCustomAttributeWithFallback(type_sizeAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (type_sizeAttr) result.getOrAddProperties().type_size = type_sizeAttr; { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(odsBuildableType0); return ::mlir::success(); } void StreamingVLOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter.printStrippedAttrOrType(getTypeSizeAttr()); ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("type_size"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); } void StreamingVLOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::StreamingVLOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::StreamingVLOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SuMopa4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SuMopa4WayOpGenericAdaptorBase::SuMopa4WayOpGenericAdaptorBase(SuMopa4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SuMopa4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SuMopa4WayOpAdaptor::SuMopa4WayOpAdaptor(SuMopa4WayOp op) : SuMopa4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SuMopa4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SuMopa4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SuMopa4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SuMopa4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SuMopa4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SuMopa4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SuMopa4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SuMopa4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SuMopa4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SuMopa4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SuMopa4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SuMopa4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SuMopa4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SuMopa4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SuMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SuMopa4WayOp SuMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMopa4WayOp SuMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SuMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SuMopa4WayOp SuMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMopa4WayOp SuMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SuMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SuMopa4WayOp SuMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMopa4WayOp SuMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SuMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SuMopa4WayOp SuMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMopa4WayOp SuMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SuMopa4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult SuMopa4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SuMopa4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SuMopa4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SuMopa4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SuMopa4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SuMopa4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::SuMops4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { SuMops4WayOpGenericAdaptorBase::SuMops4WayOpGenericAdaptorBase(SuMops4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair SuMops4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail SuMops4WayOpAdaptor::SuMops4WayOpAdaptor(SuMops4WayOp op) : SuMops4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult SuMops4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair SuMops4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange SuMops4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SuMops4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange SuMops4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult SuMops4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute SuMops4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code SuMops4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional SuMops4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void SuMops4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void SuMops4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult SuMops4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult SuMops4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void SuMops4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void SuMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } SuMops4WayOp SuMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMops4WayOp SuMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SuMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } SuMops4WayOp SuMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMops4WayOp SuMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void SuMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } SuMops4WayOp SuMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMops4WayOp SuMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void SuMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } SuMops4WayOp SuMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } SuMops4WayOp SuMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult SuMops4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult SuMops4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult SuMops4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void SuMops4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void SuMops4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::SuMops4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::SuMops4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::TileLoadOp definitions //===----------------------------------------------------------------------===// namespace detail { TileLoadOpGenericAdaptorBase::TileLoadOpGenericAdaptorBase(TileLoadOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TileLoadOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::arm_sme::TileSliceLayoutAttr TileLoadOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout TileLoadOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail TileLoadOpAdaptor::TileLoadOpAdaptor(TileLoadOp op) : TileLoadOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TileLoadOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.tile_load' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } std::pair TileLoadOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange TileLoadOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(1); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(1u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TileLoadOp::getPaddingMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TileLoadOp::getMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult TileLoadOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TileLoadOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TileLoadOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional TileLoadOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void TileLoadOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void TileLoadOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult TileLoadOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult TileLoadOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void TileLoadOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::arm_sme::TileSliceLayout TileLoadOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void TileLoadOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, VectorType resultType, Value base, ValueRange indices, TileSliceLayout layout) { build(odsBuilder, odsState, resultType, base, indices, {}, {}, layout); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, VectorType resultType, Value base, ValueRange indices, TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(base), std::forward(indices), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, VectorType resultType, Value base, ValueRange indices, TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(base), std::forward(indices), std::forward(layout)); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, VectorType resultType, Value base, ValueRange indices) { build(odsBuilder, odsState, resultType, base, indices, {}, {}, {}); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, VectorType resultType, Value base, ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultType), std::forward(base), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, VectorType resultType, Value base, ValueRange indices) { return create(builder, builder.getLoc(), std::forward(resultType), std::forward(base), std::forward(indices)); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(base); odsState.addOperands(indices); if (padding) odsState.addOperands(padding); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, static_cast(indices.size()), (padding ? 1 : 0), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout) { odsState.getOrAddProperties().layout = layout; } odsState.addTypes(result); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(base); odsState.addOperands(indices); if (padding) odsState.addOperands(padding); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, static_cast(indices.size()), (padding ? 1 : 0), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(base); odsState.addOperands(indices); if (padding) odsState.addOperands(padding); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, static_cast(indices.size()), (padding ? 1 : 0), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); odsState.addTypes(result); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(result), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); } void TileLoadOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(base); odsState.addOperands(indices); if (padding) odsState.addOperands(padding); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, static_cast(indices.size()), (padding ? 1 : 0), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value padding, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(base), std::forward(indices), std::forward(padding), std::forward(mask), std::forward(layout)); } void TileLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TileLoadOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 1u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TileLoadOp TileLoadOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileLoadOp TileLoadOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void TileLoadOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult TileLoadOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps15(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSResults(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSResults(0).begin())))))) return emitOpError("failed to verify that all of {result, base} have same element type"); if (!((!getResult() || !getPadding() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()).getElementType(), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that padding type matches element type of result"); if (!((!getResult() || !getMask() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that mask has i1 element type and same shape as result"); if (!((bool(getPadding()) == bool(getMask())))) return emitOpError("failed to verify that both `padding` and `mask` should be provided or neither"); return ::mlir::success(); } ::llvm::LogicalResult TileLoadOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult TileLoadOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand baseRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> baseOperands(&baseRawOperand, 1); ::llvm::SMLoc baseOperandsLoc; (void)baseOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> paddingOperands; ::llvm::SMLoc paddingOperandsLoc; (void)paddingOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> maskOperands; ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type baseRawType{}; ::llvm::ArrayRef<::mlir::Type> baseTypes(&baseRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); baseOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(baseRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { { paddingOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); paddingOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { maskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); maskOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); baseRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, static_cast(indicesOperands.size()), static_cast(paddingOperands.size()), static_cast(maskOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({16, 16})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({1, 1})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(128)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isF16()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::BFloat16Type>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isF64()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be a vector type that fits into a SME tile, but got " << type; } } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); result.addTypes(resultTypes); if (parser.resolveOperands(baseOperands, baseTypes, baseOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); if (!paddingOperands.empty()) { if (parser.resolveOperands(paddingOperands, ::llvm::cast(resultTypes[0]).getElementType(), paddingOperandsLoc, result.operands)) return ::mlir::failure(); } if (!maskOperands.empty()) { if (parser.resolveOperands(maskOperands, ::llvm::cast(resultTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), maskOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void TileLoadOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getBase(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; if (getMask()) { _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getPadding()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getMask()) _odsPrinter << value; } if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBase().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void TileLoadOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(0); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Read::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::TileLoadOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::TileLoadOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::TileStoreOp definitions //===----------------------------------------------------------------------===// namespace detail { TileStoreOpGenericAdaptorBase::TileStoreOpGenericAdaptorBase(TileStoreOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair TileStoreOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::arm_sme::TileSliceLayoutAttr TileStoreOpGenericAdaptorBase::getLayoutAttr() { auto attr = ::llvm::dyn_cast_or_null<::mlir::arm_sme::TileSliceLayoutAttr>(getProperties().layout); return attr; } ::mlir::arm_sme::TileSliceLayout TileStoreOpGenericAdaptorBase::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } } // namespace detail TileStoreOpAdaptor::TileStoreOpAdaptor(TileStoreOp op) : TileStoreOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult TileStoreOpAdaptor::verify(::mlir::Location loc) { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (tblgen_layout && !((::llvm::isa<::mlir::arm_sme::TileSliceLayoutAttr>(tblgen_layout)))) return emitError(loc, "'arm_sme.tile_store' op attribute 'layout' failed to satisfy constraint: Layout of a tile slice"); return ::mlir::success(); } std::pair TileStoreOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange TileStoreOp::getIndicesMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange TileStoreOp::getMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult TileStoreOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto &propStorage = prop.layout; auto attr = dict.get("layout"); if (attr) { auto convertedAttr = ::llvm::dyn_cast>(attr); if (convertedAttr) { propStorage = convertedAttr; } else { emitError() << "Invalid attribute `layout` in property conversion: " << attr; return ::mlir::failure(); } } } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute TileStoreOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.layout; if (propStorage) attrs.push_back(odsBuilder.getNamedAttr("layout", propStorage)); } { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code TileStoreOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( llvm::hash_value(prop.layout.getAsOpaquePointer()), hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional TileStoreOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "layout") return prop.layout; if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void TileStoreOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "layout") { prop.layout = ::llvm::dyn_cast_or_null>(value); return; } if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void TileStoreOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { if (prop.layout) attrs.append("layout", prop.layout); attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult TileStoreOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { { ::mlir::Attribute attr = attrs.get(getLayoutAttrName(opName)); if (attr && ::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(attr, "layout", emitError))) return ::mlir::failure(); } return ::mlir::success(); } ::llvm::LogicalResult TileStoreOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (::mlir::failed(reader.readOptionalAttribute(prop.layout))) return ::mlir::failure(); if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void TileStoreOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; writer.writeOptionalAttribute(prop.layout); if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } ::mlir::arm_sme::TileSliceLayout TileStoreOp::getLayout() { auto attr = getLayoutAttr(); return attr.getValue(); } void TileStoreOp::setLayout(::mlir::arm_sme::TileSliceLayout attrValue) { getProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::Builder((*this)->getContext()).getContext(), attrValue); } void TileStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, Value valueToStore, Value base, ValueRange indices) { build(odsBuilder, odsState, valueToStore, base, indices, {}); } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, Value valueToStore, Value base, ValueRange indices) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(valueToStore), std::forward(base), std::forward(indices)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, Value valueToStore, Value base, ValueRange indices) { return create(builder, builder.getLoc(), std::forward(valueToStore), std::forward(base), std::forward(indices)); } void TileStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(valueToStore); odsState.addOperands(base); odsState.addOperands(indices); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(indices.size()), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout) { odsState.getOrAddProperties().layout = layout; } } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); } void TileStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { odsState.addOperands(valueToStore); odsState.addOperands(base); odsState.addOperands(indices); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(indices.size()), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); if (layout) { odsState.getOrAddProperties().layout = layout; } assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayoutAttr layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); } void TileStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(valueToStore); odsState.addOperands(base); odsState.addOperands(indices); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(indices.size()), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); } void TileStoreOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { odsState.addOperands(valueToStore); odsState.addOperands(base); odsState.addOperands(indices); if (mask) odsState.addOperands(mask); ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(indices.size()), (mask ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.getOrAddProperties().layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), layout); assert(resultTypes.size() == 0u && "mismatched number of results"); odsState.addTypes(resultTypes); } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value valueToStore, ::mlir::Value base, ::mlir::ValueRange indices, /*optional*/::mlir::Value mask, ::mlir::arm_sme::TileSliceLayout layout) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(valueToStore), std::forward(base), std::forward(indices), std::forward(mask), std::forward(layout)); } void TileStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void TileStoreOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 0u && "mismatched number of return types"); odsState.addTypes(resultTypes); } TileStoreOp TileStoreOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } TileStoreOp TileStoreOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } void TileStoreOp::populateDefaultProperties(::mlir::OperationName opName, Properties &properties) { ::mlir::Builder odsBuilder(opName.getContext()); if (!properties.layout) properties.layout = ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal); } ::llvm::LogicalResult TileStoreOp::verifyInvariantsImpl() { auto tblgen_layout = getProperties().layout; (void)tblgen_layout; if (::mlir::failed(__mlir_ods_local_attr_constraint_ArmSMEOps1(*this, tblgen_layout, "layout"))) return ::mlir::failure(); { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps14(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps9(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } if (!(((getElementTypeOrSelf((*this->getODSOperands(0).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) && (getElementTypeOrSelf((*this->getODSOperands(1).begin()))) == (getElementTypeOrSelf((*this->getODSOperands(0).begin())))))) return emitOpError("failed to verify that all of {valueToStore, base} have same element type"); if (!((!getValueToStore() || !getMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that mask has i1 element type and same shape as valueToStore"); return ::mlir::success(); } ::llvm::LogicalResult TileStoreOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult TileStoreOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand valueToStoreRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> valueToStoreOperands(&valueToStoreRawOperand, 1); ::llvm::SMLoc valueToStoreOperandsLoc; (void)valueToStoreOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand baseRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> baseOperands(&baseRawOperand, 1); ::llvm::SMLoc baseOperandsLoc; (void)baseOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> indicesOperands; ::llvm::SMLoc indicesOperandsLoc; (void)indicesOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> maskOperands; ::llvm::SMLoc maskOperandsLoc; (void)maskOperandsLoc; ::mlir::arm_sme::TileSliceLayoutAttr layoutAttr; ::mlir::Type baseRawType{}; ::llvm::ArrayRef<::mlir::Type> baseTypes(&baseRawType, 1); ::mlir::Type valueToStoreRawType{}; ::llvm::ArrayRef<::mlir::Type> valueToStoreTypes(&valueToStoreRawType, 1); valueToStoreOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(valueToStoreRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); baseOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(baseRawOperand)) return ::mlir::failure(); if (parser.parseLSquare()) return ::mlir::failure(); indicesOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperandList(indicesOperands)) return ::mlir::failure(); if (parser.parseRSquare()) return ::mlir::failure(); if (::mlir::succeeded(parser.parseOptionalComma())) { { maskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); maskOperands.push_back(operand); } } } if (::mlir::succeeded(parser.parseOptionalKeyword("layout"))) { if (parser.parseCustomAttributeWithFallback(layoutAttr, ::mlir::Type{})) { return ::mlir::failure(); } if (layoutAttr) result.getOrAddProperties().layout = layoutAttr; } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::MemRefType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); baseRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); valueToStoreRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(indicesOperands.size()), static_cast(maskOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : valueToStoreTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({16, 16})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({1, 1})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(128)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (elementType.isF16()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({8, 8})))) && ([](::mlir::Type elementType) { return (::llvm::isa<::mlir::BFloat16Type>(elementType)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isF32()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isF64()); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'valueToStore' must be a vector type that fits into a SME tile, but got " << type; } } ::mlir::Type odsBuildableType0 = parser.getBuilder().getIndexType(); if (parser.resolveOperands(valueToStoreOperands, valueToStoreTypes, valueToStoreOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(baseOperands, baseTypes, baseOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(indicesOperands, odsBuildableType0, indicesOperandsLoc, result.operands)) return ::mlir::failure(); if (!maskOperands.empty()) { if (parser.resolveOperands(maskOperands, ::llvm::cast(valueToStoreTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), maskOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void TileStoreOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getValueToStore(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getBase(); _odsPrinter << "["; _odsPrinter << getIndices(); _odsPrinter << "]"; if (getMask()) { _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getMask()) _odsPrinter << value; } if (getLayoutAttr() != ::mlir::arm_sme::TileSliceLayoutAttr::get(::mlir::OpBuilder((*this)->getContext()).getContext(), TileSliceLayout::Horizontal)) { _odsPrinter << ' ' << "layout"; _odsPrinter.printStrippedAttrOrType(getLayoutAttr()); } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); elidedAttrs.push_back("layout"); { ::mlir::Builder odsBuilder(getContext()); ::mlir::Attribute attr = getLayoutAttr(); if(attr && (attr == ::mlir::arm_sme::TileSliceLayoutAttr::get(odsBuilder.getContext(), TileSliceLayout::Horizontal))) elidedAttrs.push_back("layout"); } _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getBase().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::MemRefType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getValueToStore().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void TileStoreOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { { auto valueRange = getODSOperandIndexAndLength(1); for (unsigned idx = valueRange.first; idx < valueRange.first + valueRange.second; idx++) { effects.emplace_back(::mlir::MemoryEffects::Write::get(), &getOperation()->getOpOperand(idx), 0, false, ::mlir::SideEffects::DefaultResource::get()); } } } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::TileStoreOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::TileStoreOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UMopa2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UMopa2WayOpGenericAdaptorBase::UMopa2WayOpGenericAdaptorBase(UMopa2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UMopa2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UMopa2WayOpAdaptor::UMopa2WayOpAdaptor(UMopa2WayOp op) : UMopa2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UMopa2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UMopa2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UMopa2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMopa2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMopa2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UMopa2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UMopa2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UMopa2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UMopa2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UMopa2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UMopa2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UMopa2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UMopa2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UMopa2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UMopa2WayOp UMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa2WayOp UMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMopa2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UMopa2WayOp UMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa2WayOp UMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UMopa2WayOp UMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa2WayOp UMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UMopa2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UMopa2WayOp UMopa2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa2WayOp UMopa2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UMopa2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps11(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult UMopa2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UMopa2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UMopa2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UMopa2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UMopa2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UMopa2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UMopa4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UMopa4WayOpGenericAdaptorBase::UMopa4WayOpGenericAdaptorBase(UMopa4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UMopa4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UMopa4WayOpAdaptor::UMopa4WayOpAdaptor(UMopa4WayOp op) : UMopa4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UMopa4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UMopa4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UMopa4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMopa4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMopa4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UMopa4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UMopa4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UMopa4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UMopa4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UMopa4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UMopa4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UMopa4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UMopa4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UMopa4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UMopa4WayOp UMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa4WayOp UMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UMopa4WayOp UMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa4WayOp UMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UMopa4WayOp UMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa4WayOp UMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UMopa4WayOp UMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMopa4WayOp UMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UMopa4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult UMopa4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UMopa4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UMopa4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UMopa4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UMopa4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UMopa4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UMops2WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UMops2WayOpGenericAdaptorBase::UMops2WayOpGenericAdaptorBase(UMops2WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UMops2WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UMops2WayOpAdaptor::UMops2WayOpAdaptor(UMops2WayOp op) : UMops2WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UMops2WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UMops2WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UMops2WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMops2WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMops2WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UMops2WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UMops2WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UMops2WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UMops2WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UMops2WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UMops2WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UMops2WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UMops2WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UMops2WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UMops2WayOp UMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops2WayOp UMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMops2WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UMops2WayOp UMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops2WayOp UMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UMops2WayOp UMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops2WayOp UMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UMops2WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UMops2WayOp UMops2WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops2WayOp UMops2WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UMops2WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps10(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps11(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 2)))) return emitOpError("failed to verify that tile element size equals input element size * 2"); return ::mlir::success(); } ::llvm::LogicalResult UMops2WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UMops2WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UMops2WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UMops2WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UMops2WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UMops2WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UMops4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UMops4WayOpGenericAdaptorBase::UMops4WayOpGenericAdaptorBase(UMops4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UMops4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UMops4WayOpAdaptor::UMops4WayOpAdaptor(UMops4WayOp op) : UMops4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UMops4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UMops4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UMops4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMops4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UMops4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UMops4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UMops4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UMops4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UMops4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UMops4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UMops4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UMops4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UMops4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UMops4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UMops4WayOp UMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops4WayOp UMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UMops4WayOp UMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops4WayOp UMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UMops4WayOp UMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops4WayOp UMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UMops4WayOp UMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UMops4WayOp UMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UMops4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult UMops4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UMops4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UMops4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UMops4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UMops4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UMops4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UsMopa4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UsMopa4WayOpGenericAdaptorBase::UsMopa4WayOpGenericAdaptorBase(UsMopa4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UsMopa4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UsMopa4WayOpAdaptor::UsMopa4WayOpAdaptor(UsMopa4WayOp op) : UsMopa4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UsMopa4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UsMopa4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UsMopa4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UsMopa4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UsMopa4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UsMopa4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UsMopa4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UsMopa4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UsMopa4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UsMopa4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UsMopa4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UsMopa4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UsMopa4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UsMopa4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UsMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UsMopa4WayOp UsMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMopa4WayOp UsMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UsMopa4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UsMopa4WayOp UsMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMopa4WayOp UsMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UsMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UsMopa4WayOp UsMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMopa4WayOp UsMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UsMopa4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UsMopa4WayOp UsMopa4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMopa4WayOp UsMopa4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UsMopa4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult UsMopa4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UsMopa4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UsMopa4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UsMopa4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UsMopa4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UsMopa4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::UsMops4WayOp definitions //===----------------------------------------------------------------------===// namespace detail { UsMops4WayOpGenericAdaptorBase::UsMops4WayOpGenericAdaptorBase(UsMops4WayOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} std::pair UsMops4WayOpGenericAdaptorBase::getODSOperandIndexAndLength(unsigned index, unsigned odsOperandsSize) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } } // namespace detail UsMops4WayOpAdaptor::UsMops4WayOpAdaptor(UsMops4WayOp op) : UsMops4WayOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult UsMops4WayOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } std::pair UsMops4WayOp::getODSOperandIndexAndLength(unsigned index) { ::llvm::ArrayRef sizeAttr = getProperties().operandSegmentSizes; unsigned start = 0; for (unsigned i = 0; i < index; ++i) start += sizeAttr[i]; return {start, sizeAttr[index]}; } ::mlir::MutableOperandRange UsMops4WayOp::getLhsMaskMutable() { auto range = getODSOperandIndexAndLength(2); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(2u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UsMops4WayOp::getRhsMaskMutable() { auto range = getODSOperandIndexAndLength(3); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(3u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::mlir::MutableOperandRange UsMops4WayOp::getAccMutable() { auto range = getODSOperandIndexAndLength(4); auto mutableRange = ::mlir::MutableOperandRange(getOperation(), range.first, range.second, ::mlir::MutableOperandRange::OperandSegment(4u, {getOperandSegmentSizesAttrName(), ::mlir::DenseI32ArrayAttr::get(getContext(), getProperties().operandSegmentSizes)})); return mutableRange; } ::llvm::LogicalResult UsMops4WayOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } { auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult { return convertFromAttribute(propStorage, propAttr, emitError); }; auto attr = dict.get("operandSegmentSizes"); if (!attr) attr = dict.get("operand_segment_sizes");; ; if (attr && ::mlir::failed(setFromAttr(prop.operandSegmentSizes, attr, [&]() { return emitError() << "for `operandSegmentSizes`: "; }))) return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute UsMops4WayOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; { const auto &propStorage = prop.operandSegmentSizes; auto attr = [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, propStorage); }(); attrs.push_back(odsBuilder.getNamedAttr("operandSegmentSizes", attr)); } if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code UsMops4WayOp::computePropertiesHash(const Properties &prop) { auto hash_operandSegmentSizes_ = [] (const auto &propStorage) -> llvm::hash_code { using ::llvm::hash_value; return ::llvm::hash_combine_range(std::begin(propStorage), std::end(propStorage));; }; using llvm::hash_value; return llvm::hash_combine( hash_operandSegmentSizes_(prop.operandSegmentSizes)); } std::optional UsMops4WayOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") return [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }(); return std::nullopt; } void UsMops4WayOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { if (name == "operand_segment_sizes" || name == "operandSegmentSizes") { auto arrAttr = ::llvm::dyn_cast_or_null<::mlir::DenseI32ArrayAttr>(value); if (!arrAttr) return; if (arrAttr.size() != sizeof(prop.operandSegmentSizes) / sizeof(int32_t)) return; llvm::copy(arrAttr.asArrayRef(), prop.operandSegmentSizes.begin()); return; } } void UsMops4WayOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { attrs.append("operandSegmentSizes", [&]() -> ::mlir::Attribute { return ::mlir::DenseI32ArrayAttr::get(ctx, prop.operandSegmentSizes); }()); } ::llvm::LogicalResult UsMops4WayOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } ::llvm::LogicalResult UsMops4WayOp::readProperties(::mlir::DialectBytecodeReader &reader, ::mlir::OperationState &state) { auto &prop = state.getOrAddProperties(); (void)prop; if (reader.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; ::mlir::DenseI32ArrayAttr attr; if (::mlir::failed(reader.readAttribute(attr))) return ::mlir::failure(); if (attr.size() > static_cast(sizeof(propStorage) / sizeof(int32_t))) { reader.emitError("size mismatch for operand/result_segment_size"); return ::mlir::failure(); } ::llvm::copy(::llvm::ArrayRef(attr), propStorage.begin()); } { auto &propStorage = prop.operandSegmentSizes; auto readProp = [&]() { if (reader.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) return reader.readSparseArray(::llvm::MutableArrayRef(propStorage)); ; return ::mlir::success(); }; if (::mlir::failed(readProp())) return ::mlir::failure(); } return ::mlir::success(); } void UsMops4WayOp::writeProperties(::mlir::DialectBytecodeWriter &writer) { auto &prop = getProperties(); (void)prop; if (writer.getBytecodeVersion() < /*kNativePropertiesODSSegmentSize=*/6) { auto &propStorage = prop.operandSegmentSizes; writer.writeAttribute(::mlir::DenseI32ArrayAttr::get(this->getContext(), propStorage)); } { auto &propStorage = prop.operandSegmentSizes; if (writer.getBytecodeVersion() >= /*kNativePropertiesODSSegmentSize=*/6) writer.writeSparseArray(::llvm::ArrayRef(propStorage)); ; } } void UsMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); odsState.addTypes(result); } UsMops4WayOp UsMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMops4WayOp UsMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type result, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(result), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UsMops4WayOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { odsState.addOperands(lhs); odsState.addOperands(rhs); if (lhsMask) odsState.addOperands(lhsMask); if (rhsMask) odsState.addOperands(rhsMask); if (acc) odsState.addOperands(acc); ::llvm::copy(::llvm::ArrayRef({1, 1, (lhsMask ? 1 : 0), (rhsMask ? 1 : 0), (acc ? 1 : 0)}), odsState.getOrAddProperties().operandSegmentSizes.begin()); assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } UsMops4WayOp UsMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMops4WayOp UsMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::Value lhs, ::mlir::Value rhs, /*optional*/::mlir::Value lhsMask, /*optional*/::mlir::Value rhsMask, /*optional*/::mlir::Value acc) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(lhs), std::forward(rhs), std::forward(lhsMask), std::forward(rhsMask), std::forward(acc)); } void UsMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); if (!attributes.empty()) { (void)odsState.getOrAddProperties(); ::mlir::PropertyRef properties = odsState.getRawProperties(); std::optional<::mlir::RegisteredOperationName> info = odsState.name.getRegisteredInfo(); if (failed(info->setOpPropertiesFromAttribute(odsState.name, properties, odsState.attributes.getDictionary(odsState.getContext()), nullptr))) ::llvm::report_fatal_error("Property conversion failed."); } } UsMops4WayOp UsMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMops4WayOp UsMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void UsMops4WayOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() >= 2u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } UsMops4WayOp UsMops4WayOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } UsMops4WayOp UsMops4WayOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult UsMops4WayOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSOperands(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps12(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup1 = getODSOperands(1); for (auto v : valueGroup1) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup2 = getODSOperands(2); if (valueGroup2.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup2.size(); } for (auto v : valueGroup2) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup3 = getODSOperands(3); if (valueGroup3.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup3.size(); } for (auto v : valueGroup3) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } auto valueGroup4 = getODSOperands(4); if (valueGroup4.size() > 1) { return emitOpError("operand group starting at #") << index << " requires 0 or 1 element, but found " << valueGroup4.size(); } for (auto v : valueGroup4) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps5(*this, v.getType(), "operand", index++))) return ::mlir::failure(); } } { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps13(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } if (!((((*this->getODSOperands(0).begin()).getType()) == ((*this->getODSOperands(1).begin()).getType()) && ((*this->getODSOperands(1).begin()).getType()) == ((*this->getODSOperands(0).begin()).getType())))) return emitOpError("failed to verify that all of {lhs, rhs} have same type"); if (!((!getLhs() || !getLhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(0).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(2).begin()).getType())))) return emitOpError("failed to verify that lhsMask has i1 element type and same shape as lhs"); if (!((!getRhs() || !getRhsMask() || std::equal_to<>()(::llvm::cast((*this->getODSOperands(1).begin()).getType()).cloneWith({}, IntegerType::get(this->getOperation()->getContext(), 1)), (*this->getODSOperands(3).begin()).getType())))) return emitOpError("failed to verify that rhsMask has i1 element type and same shape as rhs"); if (!((bool(getLhsMask()) == bool(getRhsMask())))) return emitOpError("failed to verify that both `lhsMask` and `rhsMask` should be provided or neither"); if (!((!getResult() || !getAcc() || std::equal_to<>()(::llvm::cast((*this->getODSResults(0).begin()).getType()), (*this->getODSOperands(4).begin()).getType())))) return emitOpError("failed to verify that `result` and `acc` have the same type"); if (!((getTileType().getElementTypeBitWidth() == (getLhsType().getElementTypeBitWidth() * 4)))) return emitOpError("failed to verify that tile element size equals input element size * 4"); return ::mlir::success(); } ::llvm::LogicalResult UsMops4WayOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult UsMops4WayOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::OpAsmParser::UnresolvedOperand lhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> lhsOperands(&lhsRawOperand, 1); ::llvm::SMLoc lhsOperandsLoc; (void)lhsOperandsLoc; ::mlir::OpAsmParser::UnresolvedOperand rhsRawOperand{}; ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> rhsOperands(&rhsRawOperand, 1); ::llvm::SMLoc rhsOperandsLoc; (void)rhsOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> accOperands; ::llvm::SMLoc accOperandsLoc; (void)accOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> lhsMaskOperands; ::llvm::SMLoc lhsMaskOperandsLoc; (void)lhsMaskOperandsLoc; ::llvm::SmallVector<::mlir::OpAsmParser::UnresolvedOperand, 4> rhsMaskOperands; ::llvm::SMLoc rhsMaskOperandsLoc; (void)rhsMaskOperandsLoc; ::mlir::Type lhsRawType{}; ::llvm::ArrayRef<::mlir::Type> lhsTypes(&lhsRawType, 1); ::mlir::Type rhsRawType{}; ::llvm::ArrayRef<::mlir::Type> rhsTypes(&rhsRawType, 1); ::mlir::Type resultRawType{}; ::llvm::ArrayRef<::mlir::Type> resultTypes(&resultRawType, 1); lhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(lhsRawOperand)) return ::mlir::failure(); if (parser.parseComma()) return ::mlir::failure(); rhsOperandsLoc = parser.getCurrentLocation(); if (parser.parseOperand(rhsRawOperand)) return ::mlir::failure(); bool accClause = false; bool masksClause = false; while(true) { if (succeeded(parser.parseOptionalKeyword("acc"))) { if (accClause) { return parser.emitError(parser.getNameLoc()) << "`acc` clause can appear at most once in the expansion of the " "oilist directive"; } accClause = true; if (parser.parseLParen()) return ::mlir::failure(); { accOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); accOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else if (succeeded(parser.parseOptionalKeyword("masks"))) { if (masksClause) { return parser.emitError(parser.getNameLoc()) << "`masks` clause can appear at most once in the expansion of the " "oilist directive"; } masksClause = true; if (parser.parseLParen()) return ::mlir::failure(); { lhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); lhsMaskOperands.push_back(operand); } } if (parser.parseComma()) return ::mlir::failure(); { rhsMaskOperandsLoc = parser.getCurrentLocation(); ::mlir::OpAsmParser::UnresolvedOperand operand; ::mlir::OptionalParseResult parseResult = parser.parseOptionalOperand(operand); if (parseResult.has_value()) { if (failed(*parseResult)) return ::mlir::failure(); rhsMaskOperands.push_back(operand); } } if (parser.parseRParen()) return ::mlir::failure(); } else { break; } } { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); lhsRawType = type; } if (parser.parseComma()) return ::mlir::failure(); { ::mlir::VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); rhsRawType = type; } if (parser.parseKeyword("into")) return ::mlir::failure(); { ::mlir::Type type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resultRawType = type; } ::llvm::copy(::llvm::ArrayRef({1, 1, static_cast(lhsMaskOperands.size()), static_cast(rhsMaskOperands.size()), static_cast(accOperands.size())}), result.getOrAddProperties().operandSegmentSizes.begin()); for (::mlir::Type type : lhsTypes) { (void)type; if (!(((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(8)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 16)))) || ((((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 1))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(16)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) && (((::llvm::isa<::mlir::ScalableVectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getNumElements() == 8)))))) { return parser.emitError(parser.getNameLoc()) << "'lhs' must be of ranks 1scalable vector of 8-bit signless integer values of length 16 or of ranks 1scalable vector of 16-bit signless integer values of length 8, but got " << type; } } for (::mlir::Type type : rhsTypes) { (void)type; if (!((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ([](::mlir::Type elementType) { return !((::llvm::isa<::mlir::TokenType>(elementType))); }(::llvm::cast<::mlir::ShapedType>(type).getElementType())))) { return parser.emitError(parser.getNameLoc()) << "'rhs' must be vector of any non-token type values, but got " << type; } } for (::mlir::Type type : resultTypes) { (void)type; if (!(((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({4, 4})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(32)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))) || ((((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() == 2))) && ((((::llvm::isa<::mlir::VectorType>(type))) && ((::llvm::cast<::mlir::VectorType>(type).getRank() > 0))) && ((::llvm::cast<::mlir::VectorType>(type).allDimsScalable()))) && ((::llvm::cast<::mlir::VectorType>(type).getShape() == ::llvm::ArrayRef({2, 2})))) && ([](::mlir::Type elementType) { return (elementType.isSignlessInteger(64)); }(::llvm::cast<::mlir::ShapedType>(type).getElementType()))))) { return parser.emitError(parser.getNameLoc()) << "'result' must be vector<[4]x[4]xi32> of 32-bit signless integer values or vector<[2]x[2]xi64> of 64-bit signless integer values, but got " << type; } } result.addTypes(resultTypes); if (parser.resolveOperands(lhsOperands, lhsTypes, lhsOperandsLoc, result.operands)) return ::mlir::failure(); if (parser.resolveOperands(rhsOperands, rhsTypes, rhsOperandsLoc, result.operands)) return ::mlir::failure(); if (!lhsMaskOperands.empty()) { if (parser.resolveOperands(lhsMaskOperands, ::llvm::cast(lhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), lhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!rhsMaskOperands.empty()) { if (parser.resolveOperands(rhsMaskOperands, ::llvm::cast(rhsTypes[0]).cloneWith({}, IntegerType::get(parser.getContext(), 1)), rhsMaskOperandsLoc, result.operands)) return ::mlir::failure(); } if (!accOperands.empty()) { if (parser.resolveOperands(accOperands, ::llvm::cast(resultTypes[0]), accOperandsLoc, result.operands)) return ::mlir::failure(); } return ::mlir::success(); } void UsMops4WayOp::print(::mlir::OpAsmPrinter &_odsPrinter) { _odsPrinter << ' '; _odsPrinter << getLhs(); _odsPrinter << ","; _odsPrinter << ' '; _odsPrinter << getRhs(); if (false || getAcc()) { _odsPrinter << ' ' << "acc"; _odsPrinter << "("; if (::mlir::Value value = getAcc()) _odsPrinter << value; _odsPrinter << ")"; } if (false || getLhsMask() || getRhsMask()) { _odsPrinter << ' ' << "masks"; _odsPrinter << "("; if (::mlir::Value value = getLhsMask()) _odsPrinter << value; _odsPrinter << ","; _odsPrinter << ' '; if (::mlir::Value value = getRhsMask()) _odsPrinter << value; _odsPrinter << ")"; } ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; elidedAttrs.push_back("operandSegmentSizes"); _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getLhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ","; _odsPrinter << ' '; { auto type = getRhs().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::VectorType>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } _odsPrinter << ' ' << "into"; _odsPrinter << ' '; { auto type = getResult().getType(); if (auto validType = ::llvm::dyn_cast<::mlir::Type>(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void UsMops4WayOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::UsMops4WayOp) MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::detail::UsMops4WayOpGenericAdaptorBase::Properties) namespace mlir::arm_sme { //===----------------------------------------------------------------------===// // ::mlir::arm_sme::ZeroOp definitions //===----------------------------------------------------------------------===// namespace detail { ZeroOpGenericAdaptorBase::ZeroOpGenericAdaptorBase(ZeroOp op) : odsAttrs(op->getRawDictionaryAttrs()), odsOpName(op->getName()), properties(op.getProperties()), odsRegions(op->getRegions()) {} } // namespace detail ZeroOpAdaptor::ZeroOpAdaptor(ZeroOp op) : ZeroOpGenericAdaptor(op->getOperands(), op) {} ::llvm::LogicalResult ZeroOpAdaptor::verify(::mlir::Location loc) { return ::mlir::success(); } ::llvm::LogicalResult ZeroOp::setPropertiesFromAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { ::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr); if (!dict) { emitError() << "expected DictionaryAttr to set properties"; return ::mlir::failure(); } return ::mlir::success(); } ::mlir::Attribute ZeroOp::getPropertiesAsAttr(::mlir::MLIRContext *ctx, const Properties &prop) { ::mlir::SmallVector<::mlir::NamedAttribute> attrs; ::mlir::Builder odsBuilder{ctx}; if (!attrs.empty()) return odsBuilder.getDictionaryAttr(attrs); return {}; } llvm::hash_code ZeroOp::computePropertiesHash(const Properties &prop) { using llvm::hash_value; return llvm::hash_combine(); } std::optional ZeroOp::getInherentAttr(::mlir::MLIRContext *ctx, const Properties &prop, llvm::StringRef name) { return std::nullopt; } void ZeroOp::setInherentAttr(Properties &prop, llvm::StringRef name, mlir::Attribute value) { } void ZeroOp::populateInherentAttrs(::mlir::MLIRContext *ctx, const Properties &prop, ::mlir::NamedAttrList &attrs) { } ::llvm::LogicalResult ZeroOp::verifyInherentAttrs(::mlir::OperationName opName, ::mlir::NamedAttrList &attrs, llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) { return ::mlir::success(); } void ZeroOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type res) { odsState.addTypes(res); } ZeroOp ZeroOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::Type res) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(res)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ZeroOp ZeroOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::Type res) { return create(builder, builder.getLoc(), std::forward(res)); } void ZeroOp::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes) { assert(resultTypes.size() == 1u && "mismatched number of results"); odsState.addTypes(resultTypes); } ZeroOp ZeroOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ZeroOp ZeroOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes) { return create(builder, builder.getLoc(), std::forward(resultTypes)); } void ZeroOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.addAttributes(attributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ZeroOp ZeroOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(attributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ZeroOp ZeroOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(attributes)); } void ZeroOp::build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { assert(operands.size() == 0u && "mismatched number of parameters"); odsState.addOperands(operands); odsState.useProperties(const_cast(properties)); odsState.addAttributes(discardableAttributes); assert(resultTypes.size() == 1u && "mismatched number of return types"); odsState.addTypes(resultTypes); } ZeroOp ZeroOp::create(::mlir::OpBuilder &builder, ::mlir::Location location, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { ::mlir::OperationState __state__(location, getOperationName()); build(builder, __state__, std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); auto __res__ = ::llvm::dyn_cast(builder.create(__state__)); assert(__res__ && "builder didn't return the right type"); return __res__; } ZeroOp ZeroOp::create(::mlir::ImplicitLocOpBuilder &builder, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes) { return create(builder, builder.getLoc(), std::forward(resultTypes), std::forward(operands), std::forward(properties), std::forward(discardableAttributes)); } ::llvm::LogicalResult ZeroOp::verifyInvariantsImpl() { { unsigned index = 0; (void)index; auto valueGroup0 = getODSResults(0); for (auto v : valueGroup0) { if (::mlir::failed(__mlir_ods_local_type_constraint_ArmSMEOps1(*this, v.getType(), "result", index++))) return ::mlir::failure(); } } return ::mlir::success(); } ::llvm::LogicalResult ZeroOp::verifyInvariants() { return verifyInvariantsImpl(); } ::mlir::ParseResult ZeroOp::parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result) { ::mlir::Type resRawType{}; ::llvm::ArrayRef<::mlir::Type> resTypes(&resRawType, 1); { auto loc = parser.getCurrentLocation();(void)loc; if (parser.parseOptionalAttrDict(result.attributes)) return ::mlir::failure(); if (failed(verifyInherentAttrs(result.name, result.attributes, [&]() { return parser.emitError(loc) << "'" << result.name.getStringRef() << "' op "; }))) return ::mlir::failure(); } if (parser.parseColon()) return ::mlir::failure(); { VectorType type; if (parser.parseCustomTypeWithFallback(type)) return ::mlir::failure(); resRawType = type; } result.addTypes(resTypes); return ::mlir::success(); } void ZeroOp::print(::mlir::OpAsmPrinter &_odsPrinter) { ::llvm::SmallVector<::llvm::StringRef, 2> elidedAttrs; _odsPrinter.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs); _odsPrinter << ' ' << ":"; _odsPrinter << ' '; { auto type = getRes().getType(); if (auto validType = ::llvm::dyn_cast(type)) _odsPrinter.printStrippedAttrOrType(validType); else _odsPrinter << type; } } void ZeroOp::getEffects(::llvm::SmallVectorImpl<::mlir::SideEffects::EffectInstance<::mlir::MemoryEffects::Effect>> &effects) { } } // namespace mlir::arm_sme MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::arm_sme::ZeroOp) #endif // GET_OP_CLASSES