/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ |* *| |* Interface Declarations *| |* *| |* Automatically generated file, do not edit! *| |* *| \*===----------------------------------------------------------------------===*/ namespace mlir::linalg { /// A Linalg contraction is defined in general terms: /// 1. Has 2 input and 1 output shapes. /// 2. Has at least one reduction dimension. /// 3. Has only projected permutation indexing maps. /// 4. its body computes `u5(u1(c) + u2(u3(a) * u4(b)))` on some field /// (AddOpType, MulOpType), where u1, u2, u3, u4 and u5 represent scalar unary /// operations that may change the type (e.g. for mixed-precision). /// As a consequence, when vectorization of such an op occurs, the only special /// behavior is that the (unique) MulOpType is vectorized into a /// `vector.contract`. All other ops are handled in a generic fashion. /// In the future, we may wish to allow more input arguments and elementwise and /// constant operations that do not involve the reduction dimension(s). class ContractionOpInterface; } // namespace mlir::linalg namespace mlir::linalg { /// A convolution is defined in general terms: /// 1. Has an `image` and a `filter` operand. /// 2. Has one `output` operand. /// 3. The indexing maps of the input have expressions that satisfy /// ``` /// AffineExpr ::== AffineDimExpr | ConvolvedExpr /// ConvolvedExpr ::== MulExpr (`+` MulExpr)+ /// MulExpr ::== AffineDimExpr (`*` (AffineConstantExpr | AffineSymbolExpr))? /// ``` /// 4. The filter and the output have projected permutation maps. /// 5. Each of the loops can be qualified as one of, /// - Loop over batch dimension, /// - Loop over output image dimensions, /// - Loop over output channel dimensions, /// - Loop over convolved filter dimensions, /// - Loop over input channel dimension. class ConvolutionOpInterface; } // namespace mlir::linalg namespace mlir::linalg { /// A fill operation is defined in general terms: /// 1. Has a scalar `value` operand. /// 2. Has one `output` operand. class FillOpInterface; } // namespace mlir::linalg namespace mlir::linalg { class LinalgOp; } // namespace mlir::linalg namespace mlir::linalg { /// Interface for decomposing aggregated operations into a sequence of simpler /// ops. class AggregatedOpInterface; } // namespace mlir::linalg namespace mlir::linalg { /// A Linalg contraction is defined in general terms: /// 1. Has 2 input and 1 output shapes. /// 2. Has at least one reduction dimension. /// 3. Has only projected permutation indexing maps. /// 4. its body computes `u5(u1(c) + u2(u3(a) * u4(b)))` on some field /// (AddOpType, MulOpType), where u1, u2, u3, u4 and u5 represent scalar unary /// operations that may change the type (e.g. for mixed-precision). /// As a consequence, when vectorization of such an op occurs, the only special /// behavior is that the (unique) MulOpType is vectorized into a /// `vector.contract`. All other ops are handled in a generic fashion. /// In the future, we may wish to allow more input arguments and elementwise and /// constant operations that do not involve the reduction dimension(s). namespace detail { struct ContractionOpInterfaceInterfaceTraits { struct Concept { /// The methods defined by the interface. Value (*lhs)(const Concept *impl, ::mlir::Operation *); Value (*rhs)(const Concept *impl, ::mlir::Operation *); bool (*isRowMajorMatmul)(const Concept *impl, ::mlir::Operation *); bool (*isColumnMajorMatmul)(const Concept *impl, ::mlir::Operation *); bool (*isRowMajorBatchMatmul)(const Concept *impl, ::mlir::Operation *); bool (*isVecmat)(const Concept *impl, ::mlir::Operation *); bool (*isBatchVecmat)(const Concept *impl, ::mlir::Operation *); bool (*isMatvec)(const Concept *impl, ::mlir::Operation *); bool (*isBatchMatvec)(const Concept *impl, ::mlir::Operation *); }; template class Model : public Concept { public: using Interface = ::mlir::linalg::ContractionOpInterface; Model() : Concept{lhs, rhs, isRowMajorMatmul, isColumnMajorMatmul, isRowMajorBatchMatmul, isVecmat, isBatchVecmat, isMatvec, isBatchMatvec} {} static inline Value lhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value rhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isRowMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isColumnMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isRowMajorBatchMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isBatchVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isBatchMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class FallbackModel : public Concept { public: using Interface = ::mlir::linalg::ContractionOpInterface; FallbackModel() : Concept{lhs, rhs, isRowMajorMatmul, isColumnMajorMatmul, isRowMajorBatchMatmul, isVecmat, isBatchVecmat, isMatvec, isBatchMatvec} {} static inline Value lhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value rhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isRowMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isColumnMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isRowMajorBatchMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isBatchVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isBatchMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class ExternalModel : public FallbackModel { public: using ConcreteEntity = ConcreteOp; }; }; template struct ContractionOpInterfaceTrait; } // namespace detail class ContractionOpInterface : public ::mlir::OpInterface { public: using ::mlir::OpInterface::OpInterface; template struct Trait : public detail::ContractionOpInterfaceTrait {}; /// Returns the left-hand side operand. Value lhs(); /// Returns the right-hand side operand. Value rhs(); /// Returns whether the given op has indexing maps that correspond to a /// row-major matmul operation. bool isRowMajorMatmul(); /// Returns whether the given op has indexing maps that correspond to a /// column-major matmul operation. bool isColumnMajorMatmul(); /// Returns whether the given op has indexing maps that correspond to a /// row-major batch matmul operation. bool isRowMajorBatchMatmul(); /// Returns whether the given op has indexing maps that correspond to a /// vector-matrix multiplication. bool isVecmat(); /// Returns whether the given op has indexing maps that correspond to a /// batched vector-matrix multiplication. bool isBatchVecmat(); /// Returns whether the given op has indexing maps that correspond to a /// matrix-vector multiplication. bool isMatvec(); /// Returns whether the given op has indexing maps that correspond to a /// batched matrix-vector multiplication. bool isBatchMatvec(); }; } // namespace mlir::linalg namespace mlir::linalg { /// A convolution is defined in general terms: /// 1. Has an `image` and a `filter` operand. /// 2. Has one `output` operand. /// 3. The indexing maps of the input have expressions that satisfy /// ``` /// AffineExpr ::== AffineDimExpr | ConvolvedExpr /// ConvolvedExpr ::== MulExpr (`+` MulExpr)+ /// MulExpr ::== AffineDimExpr (`*` (AffineConstantExpr | AffineSymbolExpr))? /// ``` /// 4. The filter and the output have projected permutation maps. /// 5. Each of the loops can be qualified as one of, /// - Loop over batch dimension, /// - Loop over output image dimensions, /// - Loop over output channel dimensions, /// - Loop over convolved filter dimensions, /// - Loop over input channel dimension. namespace detail { struct ConvolutionOpInterfaceInterfaceTraits { struct Concept { /// The methods defined by the interface. Value (*image)(const Concept *impl, ::mlir::Operation *); Value (*filter)(const Concept *impl, ::mlir::Operation *); }; template class Model : public Concept { public: using Interface = ::mlir::linalg::ConvolutionOpInterface; Model() : Concept{image, filter} {} static inline Value image(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value filter(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class FallbackModel : public Concept { public: using Interface = ::mlir::linalg::ConvolutionOpInterface; FallbackModel() : Concept{image, filter} {} static inline Value image(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value filter(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class ExternalModel : public FallbackModel { public: using ConcreteEntity = ConcreteOp; Value image(::mlir::Operation *tablegen_opaque_val) const; Value filter(::mlir::Operation *tablegen_opaque_val) const; }; }; template struct ConvolutionOpInterfaceTrait; } // namespace detail class ConvolutionOpInterface : public ::mlir::OpInterface { public: using ::mlir::OpInterface::OpInterface; template struct Trait : public detail::ConvolutionOpInterfaceTrait {}; /// Return the image operand. Value image(); /// Return the filter operand. Value filter(); }; } // namespace mlir::linalg namespace mlir::linalg { /// A fill operation is defined in general terms: /// 1. Has a scalar `value` operand. /// 2. Has one `output` operand. namespace detail { struct FillOpInterfaceInterfaceTraits { struct Concept { /// The methods defined by the interface. Value (*value)(const Concept *impl, ::mlir::Operation *); Value (*output)(const Concept *impl, ::mlir::Operation *); Value (*result)(const Concept *impl, ::mlir::Operation *); }; template class Model : public Concept { public: using Interface = ::mlir::linalg::FillOpInterface; Model() : Concept{value, output, result} {} static inline Value value(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value output(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value result(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class FallbackModel : public Concept { public: using Interface = ::mlir::linalg::FillOpInterface; FallbackModel() : Concept{value, output, result} {} static inline Value value(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value output(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Value result(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class ExternalModel : public FallbackModel { public: using ConcreteEntity = ConcreteOp; Value value(::mlir::Operation *tablegen_opaque_val) const; Value output(::mlir::Operation *tablegen_opaque_val) const; Value result(::mlir::Operation *tablegen_opaque_val) const; }; }; template struct FillOpInterfaceTrait; } // namespace detail class FillOpInterface : public ::mlir::OpInterface { public: using ::mlir::OpInterface::OpInterface; template struct Trait : public detail::FillOpInterfaceTrait {}; /// Return the fill value. Value value(); /// Return the output operand. Value output(); /// Return the result. Value result(); }; } // namespace mlir::linalg namespace mlir::linalg { namespace detail { struct LinalgOpInterfaceTraits { struct Concept { /// The methods defined by the interface. unsigned (*getNumParallelLoops)(const Concept *impl, ::mlir::Operation *); bool (*isAllParallelLoops)(const Concept *impl, ::mlir::Operation *); void (*getParallelDims)(const Concept *impl, ::mlir::Operation *, SmallVectorImpl &); unsigned (*getNumReductionLoops)(const Concept *impl, ::mlir::Operation *); void (*getReductionDims)(const Concept *impl, ::mlir::Operation *, SmallVectorImpl &); unsigned (*getNumLoops)(const Concept *impl, ::mlir::Operation *); bool (*hasSingleReductionLoop)(const Concept *impl, ::mlir::Operation *); bool (*payloadUsesValueFromOperand)(const Concept *impl, ::mlir::Operation *, OpOperand *); bool (*isSingleInputOutput)(const Concept *impl, ::mlir::Operation *); bool (*isInitTensor)(const Concept *impl, ::mlir::Operation *, OpOperand *); int64_t (*getRank)(const Concept *impl, ::mlir::Operation *, OpOperand*); Block::BlockArgListType (*getRegionInputArgs)(const Concept *impl, ::mlir::Operation *); Block::BlockArgListType (*getRegionOutputArgs)(const Concept *impl, ::mlir::Operation *); BlockArgument (*getMatchingBlockArgument)(const Concept *impl, ::mlir::Operation *, OpOperand *); OpOperand *(*getMatchingOpOperand)(const Concept *impl, ::mlir::Operation *, BlockArgument); AffineMap (*getIndexingMapMatchingResult)(const Concept *impl, ::mlir::Operation *, OpResult); OpOperand *(*getMatchingYieldValue)(const Concept *impl, ::mlir::Operation *, OpOperand*); Block*(*getBlock)(const Concept *impl, ::mlir::Operation *); SmallVector (*getIteratorTypesArray)(const Concept *impl, ::mlir::Operation *); bool (*hasDynamicIndexingMaps)(const Concept *impl, ::mlir::Operation *); LogicalResult (*verifyIndexingMapRequiredAttributes)(const Concept *impl, ::mlir::Operation *); bool (*hasDynamicShape)(const Concept *impl, ::mlir::Operation *); std::string (*getLibraryCallName)(const Concept *impl, ::mlir::Operation *); bool (*hasIndexSemantics)(const Concept *impl, ::mlir::Operation *); ::llvm::SmallVector (*getOpOperandsMatchingBBargs)(const Concept *impl, ::mlir::Operation *); LogicalResult (*mapIterationSpaceDimToOperandDim)(const Concept *impl, ::mlir::Operation *, unsigned, ::mlir::Value &, unsigned &); void (*mapIterationSpaceDimToAllOperandDims)(const Concept *impl, ::mlir::Operation *, unsigned, mlir::SmallVectorImpl>&); bool (*hasUserDefinedMaps)(const Concept *impl, ::mlir::Operation *); bool (*canOpOperandsBeDropped)(const Concept *impl, ::mlir::Operation *, ArrayRef); ArrayRef (*getShape)(const Concept *impl, ::mlir::Operation *, OpOperand*); std::function, function_ref)> (*getRegionBuilder)(); bool (*hasOnlyProjectedPermutations)(const Concept *impl, ::mlir::Operation *); /// The base classes of this interface. const ::mlir::DestinationStyleOpInterface::Concept *implDestinationStyleOpInterface = nullptr; const ::mlir::IndexingMapOpInterface::Concept *implIndexingMapOpInterface = nullptr; void initializeInterfaceConcept(::mlir::detail::InterfaceMap &interfaceMap) { implDestinationStyleOpInterface = interfaceMap.lookup<::mlir::DestinationStyleOpInterface>(); assert(implDestinationStyleOpInterface && "`::mlir::linalg::LinalgOp` expected its base interface `::mlir::DestinationStyleOpInterface` to be registered"); implIndexingMapOpInterface = interfaceMap.lookup<::mlir::IndexingMapOpInterface>(); assert(implIndexingMapOpInterface && "`::mlir::linalg::LinalgOp` expected its base interface `::mlir::IndexingMapOpInterface` to be registered"); } }; template class Model : public Concept { public: using Interface = ::mlir::linalg::LinalgOp; Model() : Concept{getNumParallelLoops, isAllParallelLoops, getParallelDims, getNumReductionLoops, getReductionDims, getNumLoops, hasSingleReductionLoop, payloadUsesValueFromOperand, isSingleInputOutput, isInitTensor, getRank, getRegionInputArgs, getRegionOutputArgs, getMatchingBlockArgument, getMatchingOpOperand, getIndexingMapMatchingResult, getMatchingYieldValue, getBlock, getIteratorTypesArray, hasDynamicIndexingMaps, verifyIndexingMapRequiredAttributes, hasDynamicShape, getLibraryCallName, hasIndexSemantics, getOpOperandsMatchingBBargs, mapIterationSpaceDimToOperandDim, mapIterationSpaceDimToAllOperandDims, hasUserDefinedMaps, canOpOperandsBeDropped, getShape, getRegionBuilder, hasOnlyProjectedPermutations} {} static inline unsigned getNumParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isAllParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline void getParallelDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res); static inline unsigned getNumReductionLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline void getReductionDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res); static inline unsigned getNumLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasSingleReductionLoop(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool payloadUsesValueFromOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline bool isSingleInputOutput(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isInitTensor(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline int64_t getRank(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline Block::BlockArgListType getRegionInputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Block::BlockArgListType getRegionOutputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline BlockArgument getMatchingBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline OpOperand *getMatchingOpOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument); static inline AffineMap getIndexingMapMatchingResult(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpResult result); static inline OpOperand *getMatchingYieldValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline Block*getBlock(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline SmallVector getIteratorTypesArray(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasDynamicIndexingMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline LogicalResult verifyIndexingMapRequiredAttributes(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasDynamicShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline std::string getLibraryCallName(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasIndexSemantics(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline ::llvm::SmallVector getOpOperandsMatchingBBargs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline LogicalResult mapIterationSpaceDimToOperandDim(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos); static inline void mapIterationSpaceDimToAllOperandDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs); static inline bool hasUserDefinedMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool canOpOperandsBeDropped(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands); static inline ArrayRef getShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline std::function, function_ref)> getRegionBuilder(); static inline bool hasOnlyProjectedPermutations(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class FallbackModel : public Concept { public: using Interface = ::mlir::linalg::LinalgOp; FallbackModel() : Concept{getNumParallelLoops, isAllParallelLoops, getParallelDims, getNumReductionLoops, getReductionDims, getNumLoops, hasSingleReductionLoop, payloadUsesValueFromOperand, isSingleInputOutput, isInitTensor, getRank, getRegionInputArgs, getRegionOutputArgs, getMatchingBlockArgument, getMatchingOpOperand, getIndexingMapMatchingResult, getMatchingYieldValue, getBlock, getIteratorTypesArray, hasDynamicIndexingMaps, verifyIndexingMapRequiredAttributes, hasDynamicShape, getLibraryCallName, hasIndexSemantics, getOpOperandsMatchingBBargs, mapIterationSpaceDimToOperandDim, mapIterationSpaceDimToAllOperandDims, hasUserDefinedMaps, canOpOperandsBeDropped, getShape, getRegionBuilder, hasOnlyProjectedPermutations} {} static inline unsigned getNumParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isAllParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline void getParallelDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res); static inline unsigned getNumReductionLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline void getReductionDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res); static inline unsigned getNumLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasSingleReductionLoop(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool payloadUsesValueFromOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline bool isSingleInputOutput(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool isInitTensor(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline int64_t getRank(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline Block::BlockArgListType getRegionInputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline Block::BlockArgListType getRegionOutputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline BlockArgument getMatchingBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand); static inline OpOperand *getMatchingOpOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument); static inline AffineMap getIndexingMapMatchingResult(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpResult result); static inline OpOperand *getMatchingYieldValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline Block*getBlock(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline SmallVector getIteratorTypesArray(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasDynamicIndexingMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline LogicalResult verifyIndexingMapRequiredAttributes(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasDynamicShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline std::string getLibraryCallName(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool hasIndexSemantics(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline ::llvm::SmallVector getOpOperandsMatchingBBargs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline LogicalResult mapIterationSpaceDimToOperandDim(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos); static inline void mapIterationSpaceDimToAllOperandDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs); static inline bool hasUserDefinedMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); static inline bool canOpOperandsBeDropped(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands); static inline ArrayRef getShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand); static inline std::function, function_ref)> getRegionBuilder(); static inline bool hasOnlyProjectedPermutations(const Concept *impl, ::mlir::Operation *tablegen_opaque_val); }; template class ExternalModel : public FallbackModel { public: using ConcreteEntity = ConcreteOp; unsigned getNumParallelLoops(::mlir::Operation *tablegen_opaque_val) const; bool isAllParallelLoops(::mlir::Operation *tablegen_opaque_val) const; void getParallelDims(::mlir::Operation *tablegen_opaque_val, SmallVectorImpl &res) const; unsigned getNumReductionLoops(::mlir::Operation *tablegen_opaque_val) const; void getReductionDims(::mlir::Operation *tablegen_opaque_val, SmallVectorImpl &res) const; unsigned getNumLoops(::mlir::Operation *tablegen_opaque_val) const; bool hasSingleReductionLoop(::mlir::Operation *tablegen_opaque_val) const; bool payloadUsesValueFromOperand(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const; bool isSingleInputOutput(::mlir::Operation *tablegen_opaque_val) const; bool isInitTensor(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const; int64_t getRank(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const; Block::BlockArgListType getRegionInputArgs(::mlir::Operation *tablegen_opaque_val) const; Block::BlockArgListType getRegionOutputArgs(::mlir::Operation *tablegen_opaque_val) const; BlockArgument getMatchingBlockArgument(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const; OpOperand *getMatchingOpOperand(::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument) const; AffineMap getIndexingMapMatchingResult(::mlir::Operation *tablegen_opaque_val, OpResult result) const; OpOperand *getMatchingYieldValue(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const; Block*getBlock(::mlir::Operation *tablegen_opaque_val) const; SmallVector getIteratorTypesArray(::mlir::Operation *tablegen_opaque_val) const; bool hasDynamicIndexingMaps(::mlir::Operation *tablegen_opaque_val) const; LogicalResult verifyIndexingMapRequiredAttributes(::mlir::Operation *tablegen_opaque_val) const; bool hasDynamicShape(::mlir::Operation *tablegen_opaque_val) const; std::string getLibraryCallName(::mlir::Operation *tablegen_opaque_val) const; ::llvm::SmallVector getOpOperandsMatchingBBargs(::mlir::Operation *tablegen_opaque_val) const; LogicalResult mapIterationSpaceDimToOperandDim(::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value &operand, unsigned &operandDimPos) const; void mapIterationSpaceDimToAllOperandDims(::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>&operandDimPairs) const; bool hasUserDefinedMaps(::mlir::Operation *tablegen_opaque_val) const; bool canOpOperandsBeDropped(::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands) const; ArrayRef getShape(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const; }; }; template struct LinalgOpTrait; } // namespace detail class LinalgOp : public ::mlir::OpInterface { public: using ::mlir::OpInterface::OpInterface; template struct Trait : public detail::LinalgOpTrait {}; /// Return the number of parallel loops. unsigned getNumParallelLoops(); /// Return true if all loops are parallel. bool isAllParallelLoops(); /// Return the dims that are parallel loops. void getParallelDims(SmallVectorImpl & res); /// Return the number of reduction loops. unsigned getNumReductionLoops(); /// Return the dims that are reduction loops. void getReductionDims(SmallVectorImpl & res); /// Return the total number of loops within the current operation. unsigned getNumLoops(); /// Returns true if the current operation has only one loop and it's a /// reduction loop. bool hasSingleReductionLoop(); /// Return true if the payload uses the value loaded from `opOperand`. This /// is useful to avoid loading from "write-only" memory that may be /// uninitialized, as well as properly cloning "read-write" operands. bool payloadUsesValueFromOperand(OpOperand * opOperand); /// Returns true only if linalgOp takes one input and produces one result. bool isSingleInputOutput(); /// Return true if `opOperand` is an init tensor. This is true when it is /// an output tensor operand whose value is used in the payload region. bool isInitTensor(OpOperand * opOperand); /// Return the `opOperand` rank or zero for scalars or vectors not wrapped within a tensor or a memref. int64_t getRank(OpOperand* opOperand); /// Return the input block arguments of the region. Block::BlockArgListType getRegionInputArgs(); /// Return the output block arguments of the region. Block::BlockArgListType getRegionOutputArgs(); /// Return the block argument for an `opOperand`. BlockArgument getMatchingBlockArgument(OpOperand * opOperand); /// Return the operand for a `blockArgument`. OpOperand *getMatchingOpOperand(BlockArgument blockArgument); /// Return the indexing map for a `result`. AffineMap getIndexingMapMatchingResult(OpResult result); /// Return the value yielded by the region corresponding to an output /// `opOperand`. OpOperand *getMatchingYieldValue(OpOperand* opOperand); /// Return the single block constituting the body of the operation by /// calling the getBody method on the concrete operation. Block*getBlock(); /// Return iterator types in the current operation. /// /// Default implementation assumes that the operation has an attribute /// `iterator_types`, but it's not always the case. Sometimes iterator types /// can be infered from other parameters and in such cases default /// getIteratorTypesArray should be overriden. SmallVector getIteratorTypesArray(); /// Return true if the indexing map is depending on the current op instance. /// This means that the indexing map is dynamically synthesized by using the /// op instance's concrete attributes, instead of being static for all /// instances of the same op kind. bool hasDynamicIndexingMaps(); /// Verify all attributes used by indexing maps are valid. LogicalResult verifyIndexingMapRequiredAttributes(); /// Return true if any of the operands has a dynamic shape. bool hasDynamicShape(); /// Return the name registered for this op when lowering to an external /// library call. std::string getLibraryCallName(); /// Return whether the op accesses the iteration indices. bool hasIndexSemantics(); /// Return op operands that have a corresponding argument in the basic block. /// By default, the block should have an argument for each operand, but there /// are expection. For example, in `map` output operand isn't used in /// the block. ::llvm::SmallVector getOpOperandsMatchingBBargs(); /// Given a dimension of the iteration space of a Linalg operation, finds an /// operand in the operation that is defined on such dimension. Returns /// whether such operand was found or not. If found, also returns the /// operand value and the dimension position within the operand. LogicalResult mapIterationSpaceDimToOperandDim(unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos); /// Given a dimension of the iteration space of a Linalg operation, finds /// all the operands in the operation that are defined on such dimension. /// Returns all the operand values found and their dimension positions in /// `operandDimPairs`. void mapIterationSpaceDimToAllOperandDims(unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs); /// Returns true if the user has supplied explicit indexing maps that are /// different from default indexing maps for this op. Returns `false` otherwise. /// Note, if the user define maps that are identical to the default maps, /// this method returns `false`. bool hasUserDefinedMaps(); /// Checks if the given operands can be dropped, and the remaining /// operands can still compute the bounds of the op. bool canOpOperandsBeDropped(ArrayRef droppedOperands); /// Return the `opOperand` shape or an empty vector for scalars or vectors /// not wrapped within a tensor or a memref. ArrayRef getShape(OpOperand* opOperand); /// Returns the region builder for constructing the body for linalg.generic. /// Returns a null function if this named op does not define a region /// builder. std::function, function_ref)> getRegionBuilder(); /// Return true if all the indexing maps are projected permutations. /// Otherwise return false. bool hasOnlyProjectedPermutations(); /// Return the flat list of all operand dimension sizes in the order they /// appear in the operands. SmallVector createFlatListOfOperandDims(OpBuilder &, Location); /// Return the flat list of all operands' static dimension sizes in the /// order they appear in the operands. All operand dimension sizes have to /// be statically known. SmallVector createFlatListOfOperandStaticDims(); /// Create the loop ranges to materialize the computation over the current /// operands. This is done by applying `getShapesToLoopsMap` to /// `createFlatListOfOperandDims`. SmallVector createLoopRanges(OpBuilder &b, Location loc); /// Returns the value that expresses the shape of the output in terms of /// shape of the input operands where possible LogicalResult reifyResultShapes(OpBuilder &b, ReifiedRankedShapedTypeDims &reifiedReturnShapes); /// Return the index in the indexingMaps vector that corresponds to this `opOperand` int64_t getIndexingMapIndex(OpOperand *opOperand); //===----------------------------------------------------------------===// // Inherited from ::mlir::DestinationStyleOpInterface //===----------------------------------------------------------------===// operator ::mlir::DestinationStyleOpInterface () const { if (!*this) return nullptr; return ::mlir::DestinationStyleOpInterface(*this, getImpl()->implDestinationStyleOpInterface); } /// Return start and end indices of the init operands range. ::mlir::MutableOperandRange getDpsInitsMutable(); ::mlir::OperandRange getDpsInits() { return (*this).getDpsInitsMutable(); } /// Return the number of DPS inits. int64_t getNumDpsInits() { return (*this).getDpsInits().size(); } /// Return the `i`-th DPS init. ::mlir::OpOperand *getDpsInitOperand(int64_t i) { return &(*this).getDpsInitsMutable()[i]; } /// Set the `i`-th DPS init. void setDpsInitOperand(int64_t i, Value value) { assert(i >= 0 && i < (*this).getNumDpsInits() && "invalid index"); (*this)->setOperand((*this).getDpsInits().getBeginOperandIndex() + i, value); } /// Return the number of DPS inputs. int64_t getNumDpsInputs() { return (*this)->getNumOperands() - (*this).getNumDpsInits(); } /// Return the DPS input operands. ::llvm::SmallVector<::mlir::OpOperand *> getDpsInputOperands() { ::llvm::SmallVector<::mlir::OpOperand *> result; int64_t numOperands = (*this)->getNumOperands(); ::mlir::OperandRange range = (*this).getDpsInits(); if (range.empty()) { result.reserve(numOperands); for (int64_t i = 0; i < numOperands; ++i) result.push_back(&(*this)->getOpOperand(i)); return result; } int64_t firstInitPos = range.getBeginOperandIndex(); int64_t numInits = range.size(); result.reserve(numOperands - numInits); for (int64_t i = 0; i < firstInitPos; ++i) result.push_back(&(*this)->getOpOperand(i)); for (int64_t i = firstInitPos + numInits; i < numOperands; ++i) result.push_back(&(*this)->getOpOperand(i)); return result; } /// Return the DPS input operands. ::llvm::SmallVector<::mlir::Value> getDpsInputs() { return ::llvm::to_vector(::llvm::map_range( (*this).getDpsInputOperands(), [](OpOperand *o) { return o->get(); })); } /// Return the `i`-th DPS input operand. ::mlir::OpOperand *getDpsInputOperand(int64_t i) { ::mlir::OperandRange range = (*this).getDpsInits(); if (range.empty()) return &(*this)->getOpOperand(i); int64_t firstInitPos = range.getBeginOperandIndex(); int64_t numInits = range.size(); assert(i >= 0 && i < (*this)->getNumOperands() - numInits && "invalid index"); return &(*this)->getOpOperand( i < firstInitPos ? i : i + firstInitPos + numInits); } /// Return "true" if `opOperand` is an "input". bool isDpsInput(::mlir::OpOperand *opOperand) { assert(opOperand->getOwner() == (*this) && "invalid operand"); return !(*this).isDpsInit(opOperand); } /// Return "true" if `opOperand` is an "init". bool isDpsInit(::mlir::OpOperand *opOperand) { assert(opOperand->getOwner() == (*this) && "invalid operand"); ::mlir::OperandRange range = (*this).getDpsInits(); if (range.empty()) return false; auto operandNumber = opOperand->getOperandNumber(); return operandNumber >= range.getBeginOperandIndex() && operandNumber < range.getBeginOperandIndex() + range.size(); } /// Return "true" if `opOperand` is a scalar value. A sclar is defined as /// neither a MemRef nor a tensor value. bool isScalar(::mlir::OpOperand *opOperand) { assert(opOperand->getOwner() == (*this) && "invalid operand"); return !::llvm::isa( opOperand->get().getType()); } /// Return the OpResult that is tied to the given OpOperand. ::mlir::OpResult getTiedOpResult(::mlir::OpOperand *opOperand) { assert(opOperand->getOwner() == (*this) && "invalid operand"); ::mlir::OperandRange range = (*this).getDpsInits(); assert(!range.empty() && "op has no inits"); int64_t resultIndex = opOperand->getOperandNumber() - range.getBeginOperandIndex(); assert(resultIndex >= 0 && resultIndex < (*this)->getNumResults()); return (*this)->getResult(resultIndex); } /// Return the OpOperand that is tied to the given OpResult. ::mlir::OpOperand *getTiedOpOperand(::mlir::OpResult opResult) { assert(opResult.getDefiningOp() == (*this) && "invalid opresult"); return (*this).getDpsInitOperand(opResult.getResultNumber()); } /// Return whether the op has pure buffer semantics. That is the case if the /// op has no tensor operands and at least one memref operand. bool hasPureBufferSemantics() { // No tensors. auto isTensor = [](Value v){ return ::llvm::isa<::mlir::TensorType>(v.getType()); }; if (::llvm::any_of((*this)->getOperands(), isTensor)) return false; // At least one memref. auto isMemref = [](Value v){ return ::llvm::isa<::mlir::BaseMemRefType>(v.getType()); }; return llvm::any_of((*this)->getOperands(), isMemref); } /// Return whether the op has pure tensor semantics. That is the case if the /// op has no memref operands and at least one tensor operand. bool hasPureTensorSemantics() { // No memrefs. auto isMemref = [](Value v){ return ::llvm::isa<::mlir::BaseMemRefType>(v.getType()); }; if (::llvm::any_of((*this)->getOperands(), isMemref)) return false; // At least one tensor. auto isTensor = [](Value v){ return ::llvm::isa<::mlir::TensorType>(v.getType()); }; return llvm::any_of((*this)->getOperands(), isTensor); } //===----------------------------------------------------------------===// // Inherited from ::mlir::IndexingMapOpInterface //===----------------------------------------------------------------===// operator ::mlir::IndexingMapOpInterface () const { if (!*this) return nullptr; return ::mlir::IndexingMapOpInterface(*this, getImpl()->implIndexingMapOpInterface); } /// Return the indexing maps attribute within the current operation. ArrayAttr getIndexingMaps(); /// Return the indexing maps within the current operation. SmallVector getIndexingMapsArray(); /// Return the input or output indexing map for `opOperand`. AffineMap getMatchingIndexingMap(OpOperand* opOperand); /// Hook to provide a custom AffineMap used to compute all the operand /// subshapes given loop bounds. This is used to answer the question: "given /// an iteration space over the codomain, what are the subshapes of the /// operands involved in the computation". /// The default behavior is to just concatenate all the indexing maps. /// A custom AffineMap allows providing a map that can be used to /// compute subshapes even in cases where the concatenation of indexing maps /// (i.e. the data traversal order) is not a simple permutation of the loop /// traversal order. It is then possible to define ops with skewed data /// traversal order for which we can still easily compute hyperrectangular /// loop bounds and subviews. AffineMap getLoopsToShapesMap(); /// Hook to provide a custom AffineMap used to construct the /// hyperrectangular loop iteration space given all the operand subshapes. /// This is used to answer the question: /// "Given a list of operand ranges, what is the subportion of the iteration /// space involved in the computation". /// This is the inverse problem of `getLoopsToShapesMap`. /// Return the empty AffineMap when such an AffineMap cannot be constructed. /// The default behavior is based on a very simple inference procedure that /// only works with permutation affine maps. /// A more advanced Tensor-Comprehension like inference is possible but has /// proven to be ambiguous in unfavorable case. /// A safer and more robust alternative is to allow each op to define /// its own AffineMap. AffineMap getShapesToLoopsMap(); /// Returns the static shape of the underlying operand (note this is /// op-specific behavior). /// Returns ShapedType::kDynamic for non-statically-known loop ranges. SmallVector getStaticOperandShape(OpOperand* opOperand); /// Returns loop ranges by composing `getShapesToLoopsMap()` with the /// flattened list of operand shapes. /// Returns ShapedType::kDynamic for non-statically-known loop ranges. SmallVector getStaticLoopRanges(); // Verifier implementation for IndexingMapOpInterface. // This must be called manually as part of other verifiers so that the // verification order, and meaningful error messages, are not preempted. LogicalResult verifyImpl(); }; } // namespace mlir::linalg namespace mlir::linalg { /// Interface for decomposing aggregated operations into a sequence of simpler /// ops. namespace detail { struct AggregatedOpInterfaceInterfaceTraits { struct Concept { /// The methods defined by the interface. FailureOr> (*decomposeOperation)(const Concept *impl, ::mlir::Operation *, OpBuilder &); }; template class Model : public Concept { public: using Interface = ::mlir::linalg::AggregatedOpInterface; Model() : Concept{decomposeOperation} {} static inline FailureOr> decomposeOperation(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpBuilder & b); }; template class FallbackModel : public Concept { public: using Interface = ::mlir::linalg::AggregatedOpInterface; FallbackModel() : Concept{decomposeOperation} {} static inline FailureOr> decomposeOperation(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpBuilder & b); }; template class ExternalModel : public FallbackModel { public: using ConcreteEntity = ConcreteOp; FailureOr> decomposeOperation(::mlir::Operation *tablegen_opaque_val, OpBuilder &b) const; }; }; template struct AggregatedOpInterfaceTrait; } // namespace detail class AggregatedOpInterface : public ::mlir::OpInterface { public: using ::mlir::OpInterface::OpInterface; template struct Trait : public detail::AggregatedOpInterfaceTrait {}; /// Method to decompose the operation into simpler operations. /// /// On success, this method returns one `Value` per result in the /// original operation. /// The order of the returned values must match the order of the /// original values. /// In other words, the returned vector can be used directly with /// `RewriterBase::replaceOp(this, returnedValues)`. FailureOr> decomposeOperation(OpBuilder & b); }; } // namespace mlir::linalg namespace mlir::linalg::detail { template struct ContractionOpInterfaceTrait : public ::mlir::OpInterface::Trait { static ::llvm::LogicalResult verifyRegionTrait(::mlir::Operation *op) { return detail::verifyContractionInterface(op); } }; } // namespace mlir::linalg::detail namespace mlir::linalg::detail { template struct ConvolutionOpInterfaceTrait : public ::mlir::OpInterface::Trait { /// Return the image operand. Value image() { return (*static_cast(this)).getOperation()->getOperand(0); } /// Return the filter operand. Value filter() { return (*static_cast(this)).getOperation()->getOperand(1); } static ::llvm::LogicalResult verifyTrait(::mlir::Operation *op) { return detail::verifyConvolutionInterface(op); } }; } // namespace mlir::linalg::detail namespace mlir::linalg::detail { template struct FillOpInterfaceTrait : public ::mlir::OpInterface::Trait { /// Return the fill value. Value value() { return (*static_cast(this)).getOperation()->getOperand(0); } /// Return the output operand. Value output() { return (*static_cast(this)).getOperation()->getOperand(1); } /// Return the result. Value result() { if ((*static_cast(this)).getOperation()->getResults().empty()) return nullptr; return (*static_cast(this)).getOperation()->getResults().front(); } static ::llvm::LogicalResult verifyTrait(::mlir::Operation *op) { return detail::verifyFillInterface(op); } }; } // namespace mlir::linalg::detail namespace mlir::linalg::detail { template struct LinalgOpTrait : public ::mlir::OpInterface::Trait { /// Return the number of parallel loops. unsigned getNumParallelLoops() { return llvm::count((*static_cast(this)).getIteratorTypesArray(), utils::IteratorType::parallel); } /// Return true if all loops are parallel. bool isAllParallelLoops() { return getNumParallelLoops() == getNumLoops(); } /// Return the dims that are parallel loops. void getParallelDims(SmallVectorImpl & res) { return findPositionsOfType((*static_cast(this)).getIteratorTypesArray(), utils::IteratorType::parallel, res); } /// Return the number of reduction loops. unsigned getNumReductionLoops() { return llvm::count((*static_cast(this)).getIteratorTypesArray(), utils::IteratorType::reduction); } /// Return the dims that are reduction loops. void getReductionDims(SmallVectorImpl & res) { return findPositionsOfType((*static_cast(this)).getIteratorTypesArray(), utils::IteratorType::reduction, res); } /// Return the total number of loops within the current operation. unsigned getNumLoops() { return (*static_cast(this)).getIteratorTypesArray().size(); } /// Returns true if the current operation has only one loop and it's a /// reduction loop. bool hasSingleReductionLoop() { auto iters = (*static_cast(this)).getIteratorTypesArray(); return iters.size() == 1 && llvm::count(iters, utils::IteratorType::reduction) == 1; } /// Return true if the payload uses the value loaded from `opOperand`. This /// is useful to avoid loading from "write-only" memory that may be /// uninitialized, as well as properly cloning "read-write" operands. bool payloadUsesValueFromOperand(OpOperand * opOperand) { unsigned bbArgNumber = opOperand->getOperandNumber(); // Init tensors have uses. return !getBlock()->getArgument(bbArgNumber).use_empty(); } /// Returns true only if linalgOp takes one input and produces one result. bool isSingleInputOutput() { return (*static_cast(this)).getNumDpsInputs() == 1 && (*static_cast(this)).getNumDpsInits() == 1; } /// Return true if `opOperand` is an init tensor. This is true when it is /// an output tensor operand whose value is used in the payload region. bool isInitTensor(OpOperand * opOperand) { if (!(*static_cast(this)).isDpsInit(opOperand)) return false; return payloadUsesValueFromOperand(opOperand); } /// Return the `opOperand` rank or zero for scalars or vectors not wrapped within a tensor or a memref. int64_t getRank(OpOperand* opOperand) { assert(opOperand->getOwner() == this->getOperation()); Type t = opOperand->get().getType(); // A VectorType is an elemental type, do not consider its rank for the operand. if (isa(t)) return 0; // Tensor and Memref container types have a rank. if (auto shapedType = ::llvm::dyn_cast(t)) { // Failsafe. assert((isa(t) || isa(t)) && "expected a ranked tensor or memref in LinalgInterface::getRank"); return shapedType.getRank(); } return 0; } /// Return the input block arguments of the region. Block::BlockArgListType getRegionInputArgs() { return getBlock()->getArguments().take_front((*static_cast(this)).getNumDpsInputs()); } /// Return the output block arguments of the region. Block::BlockArgListType getRegionOutputArgs() { return getBlock()->getArguments().take_back((*static_cast(this)).getNumDpsInits()); } /// Return the block argument for an `opOperand`. BlockArgument getMatchingBlockArgument(OpOperand * opOperand) { assert(opOperand->getOwner() == this->getOperation()); return getBlock()->getArgument(opOperand->getOperandNumber()); } /// Return the operand for a `blockArgument`. OpOperand *getMatchingOpOperand(BlockArgument blockArgument) { assert(blockArgument.getOwner() == getBlock()); return &this->getOperation()->getOpOperand( blockArgument.getArgNumber()); } /// Return the indexing map for a `result`. AffineMap getIndexingMapMatchingResult(OpResult result) { assert(result.getOwner() == this->getOperation()); auto indexingMaps = (*static_cast(this)).getIndexingMaps().template getAsValueRange(); return *(indexingMaps.begin() + (*static_cast(this)).getNumDpsInputs() + result.getResultNumber()); } /// Return the value yielded by the region corresponding to an output /// `opOperand`. OpOperand *getMatchingYieldValue(OpOperand* opOperand) { assert(opOperand->getOwner() == this->getOperation()); int64_t resultIndex = opOperand->getOperandNumber() - (*static_cast(this)).getNumDpsInputs(); assert(resultIndex >= 0 && resultIndex < (*static_cast(this)).getNumDpsInits()); Operation *yieldOp = getBlock()->getTerminator(); return &yieldOp->getOpOperand(resultIndex); } /// Return the single block constituting the body of the operation by /// calling the getBody method on the concrete operation. Block*getBlock() { // Assume the concrete operation implements the // SingleBlockImplicitTerminator trait. return (*static_cast(this)).getBody(); } /// Return iterator types in the current operation. /// /// Default implementation assumes that the operation has an attribute /// `iterator_types`, but it's not always the case. Sometimes iterator types /// can be infered from other parameters and in such cases default /// getIteratorTypesArray should be overriden. SmallVector getIteratorTypesArray() { auto range = (*static_cast(this)).getIteratorTypes() .template getAsValueRange(); return {range.begin(), range.end()}; } /// Return true if the indexing map is depending on the current op instance. /// This means that the indexing map is dynamically synthesized by using the /// op instance's concrete attributes, instead of being static for all /// instances of the same op kind. bool hasDynamicIndexingMaps() { return false; } /// Verify all attributes used by indexing maps are valid. LogicalResult verifyIndexingMapRequiredAttributes() { return success(); } /// Return true if any of the operands has a dynamic shape. bool hasDynamicShape() { for (OpOperand &opOperand : this->getOperation()->getOpOperands()) { if (auto shapedType = dyn_cast(opOperand.get().getType())) { if (ShapedType::isDynamicShape(shapedType.getShape())) return true; } } return false; } /// Return the name registered for this op when lowering to an external /// library call. std::string getLibraryCallName() { return (*static_cast(this)).getLibraryCallName(); } /// Return op operands that have a corresponding argument in the basic block. /// By default, the block should have an argument for each operand, but there /// are expection. For example, in `map` output operand isn't used in /// the block. ::llvm::SmallVector getOpOperandsMatchingBBargs() { ::llvm::SmallVector result; result.reserve((*static_cast(this))->getNumOperands()); llvm::transform( this->getOperation()->getOpOperands(), std::back_inserter(result), [](OpOperand &opOperand) { return &opOperand; }); return result; } /// Given a dimension of the iteration space of a Linalg operation, finds an /// operand in the operation that is defined on such dimension. Returns /// whether such operand was found or not. If found, also returns the /// operand value and the dimension position within the operand. LogicalResult mapIterationSpaceDimToOperandDim(unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos) { // Retrieve the operand and its dimension position from the first // operand with a permutation map that is defined on such dimension. for (auto [i, idxMap] : llvm::enumerate((*static_cast(this)).getIndexingMapsArray())) { if (idxMap.isProjectedPermutation()) { if (auto mayOperandDim = idxMap.getResultPosition( getAffineDimExpr(dimPos, idxMap.getContext()))) { operand = (*static_cast(this))->getOperand(i); operandDimPos = *mayOperandDim; return success(); } } } return failure(); } /// Given a dimension of the iteration space of a Linalg operation, finds /// all the operands in the operation that are defined on such dimension. /// Returns all the operand values found and their dimension positions in /// `operandDimPairs`. void mapIterationSpaceDimToAllOperandDims(unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs) { for (auto [i, idxMap] : llvm::enumerate((*static_cast(this)).getIndexingMapsArray())) { if (idxMap.isProjectedPermutation()) { if (auto mayOperandDim = idxMap.getResultPosition( getAffineDimExpr(dimPos, idxMap.getContext()))) { operandDimPairs.push_back({(*static_cast(this))->getOperand(i), *mayOperandDim}); } } } return; } /// Returns true if the user has supplied explicit indexing maps that are /// different from default indexing maps for this op. Returns `false` otherwise. /// Note, if the user define maps that are identical to the default maps, /// this method returns `false`. bool hasUserDefinedMaps() { return false; } /// Checks if the given operands can be dropped, and the remaining /// operands can still compute the bounds of the op. bool canOpOperandsBeDropped(ArrayRef droppedOperands) { return detail::canOpOperandsBeDroppedImpl((*static_cast(this)), droppedOperands); } /// Return the `opOperand` shape or an empty vector for scalars or vectors /// not wrapped within a tensor or a memref. ArrayRef getShape(OpOperand* opOperand) { Type t = opOperand->get().getType(); // A VectorType is an elemental type, do not consider its rank for the operand. if (isa(t)) return {}; if (auto shapedType = ::llvm::dyn_cast(t)) { // Failsafe. assert((isa(t) || isa(t)) && "expected a ranked tensor or memref in LinalgInterface::getRank"); return shapedType.getShape(); } return {}; } static ::llvm::LogicalResult verifyRegionTrait(::mlir::Operation *op) { return detail::verifyStructuredOpInterface(op); } }; } // namespace mlir::linalg::detail namespace mlir::linalg::detail { template struct AggregatedOpInterfaceTrait : public ::mlir::OpInterface::Trait { /// Method to decompose the operation into simpler operations. /// /// On success, this method returns one `Value` per result in the /// original operation. /// The order of the returned values must match the order of the /// original values. /// In other words, the returned vector can be used directly with /// `RewriterBase::replaceOp(this, returnedValues)`. FailureOr> decomposeOperation(OpBuilder & b) { return {}; } }; } // namespace mlir::linalg::detail namespace mlir::linalg { template Value detail::ContractionOpInterfaceInterfaceTraits::Model::lhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(0); } template Value detail::ContractionOpInterfaceInterfaceTraits::Model::rhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(1); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isRowMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isRowMajorMatmul((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isColumnMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isColumnMajorMatmul((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isRowMajorBatchMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isRowMajorBatchMatmul((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isVecmat((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isBatchVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isBatchVecmat((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isMatvec((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template bool detail::ContractionOpInterfaceInterfaceTraits::Model::isBatchMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return mlir::isBatchMatvec((llvm::cast(tablegen_opaque_val)).getIndexingMaps()); } template Value detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::lhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->lhs(tablegen_opaque_val); } template Value detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::rhs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->rhs(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isRowMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isRowMajorMatmul(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isColumnMajorMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isColumnMajorMatmul(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isRowMajorBatchMatmul(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isRowMajorBatchMatmul(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isVecmat(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isBatchVecmat(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isBatchVecmat(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isMatvec(tablegen_opaque_val); } template bool detail::ContractionOpInterfaceInterfaceTraits::FallbackModel::isBatchMatvec(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isBatchMatvec(tablegen_opaque_val); } } // namespace mlir::linalg namespace mlir::linalg { template Value detail::ConvolutionOpInterfaceInterfaceTraits::Model::image(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).image(); } template Value detail::ConvolutionOpInterfaceInterfaceTraits::Model::filter(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).filter(); } template Value detail::ConvolutionOpInterfaceInterfaceTraits::FallbackModel::image(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->image(tablegen_opaque_val); } template Value detail::ConvolutionOpInterfaceInterfaceTraits::FallbackModel::filter(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->filter(tablegen_opaque_val); } template Value detail::ConvolutionOpInterfaceInterfaceTraits::ExternalModel::image(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(0); } template Value detail::ConvolutionOpInterfaceInterfaceTraits::ExternalModel::filter(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(1); } } // namespace mlir::linalg namespace mlir::linalg { template Value detail::FillOpInterfaceInterfaceTraits::Model::value(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).value(); } template Value detail::FillOpInterfaceInterfaceTraits::Model::output(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).output(); } template Value detail::FillOpInterfaceInterfaceTraits::Model::result(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).result(); } template Value detail::FillOpInterfaceInterfaceTraits::FallbackModel::value(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->value(tablegen_opaque_val); } template Value detail::FillOpInterfaceInterfaceTraits::FallbackModel::output(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->output(tablegen_opaque_val); } template Value detail::FillOpInterfaceInterfaceTraits::FallbackModel::result(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->result(tablegen_opaque_val); } template Value detail::FillOpInterfaceInterfaceTraits::ExternalModel::value(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(0); } template Value detail::FillOpInterfaceInterfaceTraits::ExternalModel::output(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getOperation()->getOperand(1); } template Value detail::FillOpInterfaceInterfaceTraits::ExternalModel::result(::mlir::Operation *tablegen_opaque_val) const { if ((llvm::cast(tablegen_opaque_val)).getOperation()->getResults().empty()) return nullptr; return (llvm::cast(tablegen_opaque_val)).getOperation()->getResults().front(); } } // namespace mlir::linalg namespace mlir::linalg { template unsigned detail::LinalgOpInterfaceTraits::Model::getNumParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getNumParallelLoops(); } template bool detail::LinalgOpInterfaceTraits::Model::isAllParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).isAllParallelLoops(); } template void detail::LinalgOpInterfaceTraits::Model::getParallelDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res) { return (llvm::cast(tablegen_opaque_val)).getParallelDims(res); } template unsigned detail::LinalgOpInterfaceTraits::Model::getNumReductionLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getNumReductionLoops(); } template void detail::LinalgOpInterfaceTraits::Model::getReductionDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res) { return (llvm::cast(tablegen_opaque_val)).getReductionDims(res); } template unsigned detail::LinalgOpInterfaceTraits::Model::getNumLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getNumLoops(); } template bool detail::LinalgOpInterfaceTraits::Model::hasSingleReductionLoop(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).hasSingleReductionLoop(); } template bool detail::LinalgOpInterfaceTraits::Model::payloadUsesValueFromOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return (llvm::cast(tablegen_opaque_val)).payloadUsesValueFromOperand(opOperand); } template bool detail::LinalgOpInterfaceTraits::Model::isSingleInputOutput(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).isSingleInputOutput(); } template bool detail::LinalgOpInterfaceTraits::Model::isInitTensor(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return (llvm::cast(tablegen_opaque_val)).isInitTensor(opOperand); } template int64_t detail::LinalgOpInterfaceTraits::Model::getRank(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return (llvm::cast(tablegen_opaque_val)).getRank(opOperand); } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::Model::getRegionInputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getRegionInputArgs(); } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::Model::getRegionOutputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getRegionOutputArgs(); } template BlockArgument detail::LinalgOpInterfaceTraits::Model::getMatchingBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return (llvm::cast(tablegen_opaque_val)).getMatchingBlockArgument(opOperand); } template OpOperand *detail::LinalgOpInterfaceTraits::Model::getMatchingOpOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument) { return (llvm::cast(tablegen_opaque_val)).getMatchingOpOperand(blockArgument); } template AffineMap detail::LinalgOpInterfaceTraits::Model::getIndexingMapMatchingResult(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpResult result) { return (llvm::cast(tablegen_opaque_val)).getIndexingMapMatchingResult(result); } template OpOperand *detail::LinalgOpInterfaceTraits::Model::getMatchingYieldValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return (llvm::cast(tablegen_opaque_val)).getMatchingYieldValue(opOperand); } template Block*detail::LinalgOpInterfaceTraits::Model::getBlock(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getBlock(); } template SmallVector detail::LinalgOpInterfaceTraits::Model::getIteratorTypesArray(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(); } template bool detail::LinalgOpInterfaceTraits::Model::hasDynamicIndexingMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).hasDynamicIndexingMaps(); } template LogicalResult detail::LinalgOpInterfaceTraits::Model::verifyIndexingMapRequiredAttributes(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).verifyIndexingMapRequiredAttributes(); } template bool detail::LinalgOpInterfaceTraits::Model::hasDynamicShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).hasDynamicShape(); } template std::string detail::LinalgOpInterfaceTraits::Model::getLibraryCallName(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getLibraryCallName(); } template bool detail::LinalgOpInterfaceTraits::Model::hasIndexSemantics(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).hasIndexSemantics(); } template ::llvm::SmallVector detail::LinalgOpInterfaceTraits::Model::getOpOperandsMatchingBBargs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).getOpOperandsMatchingBBargs(); } template LogicalResult detail::LinalgOpInterfaceTraits::Model::mapIterationSpaceDimToOperandDim(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos) { return (llvm::cast(tablegen_opaque_val)).mapIterationSpaceDimToOperandDim(dimPos, operand, operandDimPos); } template void detail::LinalgOpInterfaceTraits::Model::mapIterationSpaceDimToAllOperandDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs) { return (llvm::cast(tablegen_opaque_val)).mapIterationSpaceDimToAllOperandDims(dimPos, operandDimPairs); } template bool detail::LinalgOpInterfaceTraits::Model::hasUserDefinedMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return (llvm::cast(tablegen_opaque_val)).hasUserDefinedMaps(); } template bool detail::LinalgOpInterfaceTraits::Model::canOpOperandsBeDropped(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands) { return (llvm::cast(tablegen_opaque_val)).canOpOperandsBeDropped(droppedOperands); } template ArrayRef detail::LinalgOpInterfaceTraits::Model::getShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return (llvm::cast(tablegen_opaque_val)).getShape(opOperand); } template std::function, function_ref)> detail::LinalgOpInterfaceTraits::Model::getRegionBuilder() { return ConcreteOp::getRegionBuilder(); } template bool detail::LinalgOpInterfaceTraits::Model::hasOnlyProjectedPermutations(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return llvm::all_of((llvm::cast(tablegen_opaque_val)).getIndexingMapsArray(), [](AffineMap map) { return map.isProjectedPermutation(); }); } template unsigned detail::LinalgOpInterfaceTraits::FallbackModel::getNumParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getNumParallelLoops(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::isAllParallelLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isAllParallelLoops(tablegen_opaque_val); } template void detail::LinalgOpInterfaceTraits::FallbackModel::getParallelDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res) { return static_cast(impl)->getParallelDims(tablegen_opaque_val, res); } template unsigned detail::LinalgOpInterfaceTraits::FallbackModel::getNumReductionLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getNumReductionLoops(tablegen_opaque_val); } template void detail::LinalgOpInterfaceTraits::FallbackModel::getReductionDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, SmallVectorImpl & res) { return static_cast(impl)->getReductionDims(tablegen_opaque_val, res); } template unsigned detail::LinalgOpInterfaceTraits::FallbackModel::getNumLoops(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getNumLoops(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasSingleReductionLoop(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasSingleReductionLoop(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::payloadUsesValueFromOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return static_cast(impl)->payloadUsesValueFromOperand(tablegen_opaque_val, opOperand); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::isSingleInputOutput(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->isSingleInputOutput(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::isInitTensor(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return static_cast(impl)->isInitTensor(tablegen_opaque_val, opOperand); } template int64_t detail::LinalgOpInterfaceTraits::FallbackModel::getRank(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return static_cast(impl)->getRank(tablegen_opaque_val, opOperand); } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::FallbackModel::getRegionInputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getRegionInputArgs(tablegen_opaque_val); } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::FallbackModel::getRegionOutputArgs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getRegionOutputArgs(tablegen_opaque_val); } template BlockArgument detail::LinalgOpInterfaceTraits::FallbackModel::getMatchingBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand * opOperand) { return static_cast(impl)->getMatchingBlockArgument(tablegen_opaque_val, opOperand); } template OpOperand *detail::LinalgOpInterfaceTraits::FallbackModel::getMatchingOpOperand(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument) { return static_cast(impl)->getMatchingOpOperand(tablegen_opaque_val, blockArgument); } template AffineMap detail::LinalgOpInterfaceTraits::FallbackModel::getIndexingMapMatchingResult(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpResult result) { return static_cast(impl)->getIndexingMapMatchingResult(tablegen_opaque_val, result); } template OpOperand *detail::LinalgOpInterfaceTraits::FallbackModel::getMatchingYieldValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return static_cast(impl)->getMatchingYieldValue(tablegen_opaque_val, opOperand); } template Block*detail::LinalgOpInterfaceTraits::FallbackModel::getBlock(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getBlock(tablegen_opaque_val); } template SmallVector detail::LinalgOpInterfaceTraits::FallbackModel::getIteratorTypesArray(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getIteratorTypesArray(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasDynamicIndexingMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasDynamicIndexingMaps(tablegen_opaque_val); } template LogicalResult detail::LinalgOpInterfaceTraits::FallbackModel::verifyIndexingMapRequiredAttributes(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->verifyIndexingMapRequiredAttributes(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasDynamicShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasDynamicShape(tablegen_opaque_val); } template std::string detail::LinalgOpInterfaceTraits::FallbackModel::getLibraryCallName(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getLibraryCallName(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasIndexSemantics(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasIndexSemantics(tablegen_opaque_val); } template ::llvm::SmallVector detail::LinalgOpInterfaceTraits::FallbackModel::getOpOperandsMatchingBBargs(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->getOpOperandsMatchingBBargs(tablegen_opaque_val); } template LogicalResult detail::LinalgOpInterfaceTraits::FallbackModel::mapIterationSpaceDimToOperandDim(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value & operand, unsigned & operandDimPos) { return static_cast(impl)->mapIterationSpaceDimToOperandDim(tablegen_opaque_val, dimPos, operand, operandDimPos); } template void detail::LinalgOpInterfaceTraits::FallbackModel::mapIterationSpaceDimToAllOperandDims(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>& operandDimPairs) { return static_cast(impl)->mapIterationSpaceDimToAllOperandDims(tablegen_opaque_val, dimPos, operandDimPairs); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasUserDefinedMaps(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasUserDefinedMaps(tablegen_opaque_val); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::canOpOperandsBeDropped(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands) { return static_cast(impl)->canOpOperandsBeDropped(tablegen_opaque_val, droppedOperands); } template ArrayRef detail::LinalgOpInterfaceTraits::FallbackModel::getShape(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpOperand* opOperand) { return static_cast(impl)->getShape(tablegen_opaque_val, opOperand); } template std::function, function_ref)> detail::LinalgOpInterfaceTraits::FallbackModel::getRegionBuilder() { return ConcreteOp::getRegionBuilder(); } template bool detail::LinalgOpInterfaceTraits::FallbackModel::hasOnlyProjectedPermutations(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) { return static_cast(impl)->hasOnlyProjectedPermutations(tablegen_opaque_val); } template unsigned detail::LinalgOpInterfaceTraits::ExternalModel::getNumParallelLoops(::mlir::Operation *tablegen_opaque_val) const { return llvm::count((llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(), utils::IteratorType::parallel); } template bool detail::LinalgOpInterfaceTraits::ExternalModel::isAllParallelLoops(::mlir::Operation *tablegen_opaque_val) const { return getNumParallelLoops() == getNumLoops(); } template void detail::LinalgOpInterfaceTraits::ExternalModel::getParallelDims(::mlir::Operation *tablegen_opaque_val, SmallVectorImpl &res) const { return findPositionsOfType((llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(), utils::IteratorType::parallel, res); } template unsigned detail::LinalgOpInterfaceTraits::ExternalModel::getNumReductionLoops(::mlir::Operation *tablegen_opaque_val) const { return llvm::count((llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(), utils::IteratorType::reduction); } template void detail::LinalgOpInterfaceTraits::ExternalModel::getReductionDims(::mlir::Operation *tablegen_opaque_val, SmallVectorImpl &res) const { return findPositionsOfType((llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(), utils::IteratorType::reduction, res); } template unsigned detail::LinalgOpInterfaceTraits::ExternalModel::getNumLoops(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getIteratorTypesArray().size(); } template bool detail::LinalgOpInterfaceTraits::ExternalModel::hasSingleReductionLoop(::mlir::Operation *tablegen_opaque_val) const { auto iters = (llvm::cast(tablegen_opaque_val)).getIteratorTypesArray(); return iters.size() == 1 && llvm::count(iters, utils::IteratorType::reduction) == 1; } template bool detail::LinalgOpInterfaceTraits::ExternalModel::payloadUsesValueFromOperand(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const { unsigned bbArgNumber = opOperand->getOperandNumber(); // Init tensors have uses. return !getBlock()->getArgument(bbArgNumber).use_empty(); } template bool detail::LinalgOpInterfaceTraits::ExternalModel::isSingleInputOutput(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getNumDpsInputs() == 1 && (llvm::cast(tablegen_opaque_val)).getNumDpsInits() == 1; } template bool detail::LinalgOpInterfaceTraits::ExternalModel::isInitTensor(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const { if (!(llvm::cast(tablegen_opaque_val)).isDpsInit(opOperand)) return false; return payloadUsesValueFromOperand(opOperand); } template int64_t detail::LinalgOpInterfaceTraits::ExternalModel::getRank(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const { assert(opOperand->getOwner() == this->getOperation()); Type t = opOperand->get().getType(); // A VectorType is an elemental type, do not consider its rank for the operand. if (isa(t)) return 0; // Tensor and Memref container types have a rank. if (auto shapedType = ::llvm::dyn_cast(t)) { // Failsafe. assert((isa(t) || isa(t)) && "expected a ranked tensor or memref in LinalgInterface::getRank"); return shapedType.getRank(); } return 0; } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::ExternalModel::getRegionInputArgs(::mlir::Operation *tablegen_opaque_val) const { return getBlock()->getArguments().take_front((llvm::cast(tablegen_opaque_val)).getNumDpsInputs()); } template Block::BlockArgListType detail::LinalgOpInterfaceTraits::ExternalModel::getRegionOutputArgs(::mlir::Operation *tablegen_opaque_val) const { return getBlock()->getArguments().take_back((llvm::cast(tablegen_opaque_val)).getNumDpsInits()); } template BlockArgument detail::LinalgOpInterfaceTraits::ExternalModel::getMatchingBlockArgument(::mlir::Operation *tablegen_opaque_val, OpOperand *opOperand) const { assert(opOperand->getOwner() == this->getOperation()); return getBlock()->getArgument(opOperand->getOperandNumber()); } template OpOperand *detail::LinalgOpInterfaceTraits::ExternalModel::getMatchingOpOperand(::mlir::Operation *tablegen_opaque_val, BlockArgument blockArgument) const { assert(blockArgument.getOwner() == getBlock()); return &this->getOperation()->getOpOperand( blockArgument.getArgNumber()); } template AffineMap detail::LinalgOpInterfaceTraits::ExternalModel::getIndexingMapMatchingResult(::mlir::Operation *tablegen_opaque_val, OpResult result) const { assert(result.getOwner() == this->getOperation()); auto indexingMaps = (llvm::cast(tablegen_opaque_val)).getIndexingMaps().template getAsValueRange(); return *(indexingMaps.begin() + (llvm::cast(tablegen_opaque_val)).getNumDpsInputs() + result.getResultNumber()); } template OpOperand *detail::LinalgOpInterfaceTraits::ExternalModel::getMatchingYieldValue(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const { assert(opOperand->getOwner() == this->getOperation()); int64_t resultIndex = opOperand->getOperandNumber() - (llvm::cast(tablegen_opaque_val)).getNumDpsInputs(); assert(resultIndex >= 0 && resultIndex < (llvm::cast(tablegen_opaque_val)).getNumDpsInits()); Operation *yieldOp = getBlock()->getTerminator(); return &yieldOp->getOpOperand(resultIndex); } template Block*detail::LinalgOpInterfaceTraits::ExternalModel::getBlock(::mlir::Operation *tablegen_opaque_val) const { // Assume the concrete operation implements the // SingleBlockImplicitTerminator trait. return (llvm::cast(tablegen_opaque_val)).getBody(); } template SmallVector detail::LinalgOpInterfaceTraits::ExternalModel::getIteratorTypesArray(::mlir::Operation *tablegen_opaque_val) const { auto range = (llvm::cast(tablegen_opaque_val)).getIteratorTypes() .template getAsValueRange(); return {range.begin(), range.end()}; } template bool detail::LinalgOpInterfaceTraits::ExternalModel::hasDynamicIndexingMaps(::mlir::Operation *tablegen_opaque_val) const { return false; } template LogicalResult detail::LinalgOpInterfaceTraits::ExternalModel::verifyIndexingMapRequiredAttributes(::mlir::Operation *tablegen_opaque_val) const { return success(); } template bool detail::LinalgOpInterfaceTraits::ExternalModel::hasDynamicShape(::mlir::Operation *tablegen_opaque_val) const { for (OpOperand &opOperand : this->getOperation()->getOpOperands()) { if (auto shapedType = dyn_cast(opOperand.get().getType())) { if (ShapedType::isDynamicShape(shapedType.getShape())) return true; } } return false; } template std::string detail::LinalgOpInterfaceTraits::ExternalModel::getLibraryCallName(::mlir::Operation *tablegen_opaque_val) const { return (llvm::cast(tablegen_opaque_val)).getLibraryCallName(); } template ::llvm::SmallVector detail::LinalgOpInterfaceTraits::ExternalModel::getOpOperandsMatchingBBargs(::mlir::Operation *tablegen_opaque_val) const { ::llvm::SmallVector result; result.reserve((llvm::cast(tablegen_opaque_val))->getNumOperands()); llvm::transform( this->getOperation()->getOpOperands(), std::back_inserter(result), [](OpOperand &opOperand) { return &opOperand; }); return result; } template LogicalResult detail::LinalgOpInterfaceTraits::ExternalModel::mapIterationSpaceDimToOperandDim(::mlir::Operation *tablegen_opaque_val, unsigned dimPos, ::mlir::Value &operand, unsigned &operandDimPos) const { // Retrieve the operand and its dimension position from the first // operand with a permutation map that is defined on such dimension. for (auto [i, idxMap] : llvm::enumerate((llvm::cast(tablegen_opaque_val)).getIndexingMapsArray())) { if (idxMap.isProjectedPermutation()) { if (auto mayOperandDim = idxMap.getResultPosition( getAffineDimExpr(dimPos, idxMap.getContext()))) { operand = (llvm::cast(tablegen_opaque_val))->getOperand(i); operandDimPos = *mayOperandDim; return success(); } } } return failure(); } template void detail::LinalgOpInterfaceTraits::ExternalModel::mapIterationSpaceDimToAllOperandDims(::mlir::Operation *tablegen_opaque_val, unsigned dimPos, mlir::SmallVectorImpl>&operandDimPairs) const { for (auto [i, idxMap] : llvm::enumerate((llvm::cast(tablegen_opaque_val)).getIndexingMapsArray())) { if (idxMap.isProjectedPermutation()) { if (auto mayOperandDim = idxMap.getResultPosition( getAffineDimExpr(dimPos, idxMap.getContext()))) { operandDimPairs.push_back({(llvm::cast(tablegen_opaque_val))->getOperand(i), *mayOperandDim}); } } } return; } template bool detail::LinalgOpInterfaceTraits::ExternalModel::hasUserDefinedMaps(::mlir::Operation *tablegen_opaque_val) const { return false; } template bool detail::LinalgOpInterfaceTraits::ExternalModel::canOpOperandsBeDropped(::mlir::Operation *tablegen_opaque_val, ArrayRef droppedOperands) const { return detail::canOpOperandsBeDroppedImpl((llvm::cast(tablegen_opaque_val)), droppedOperands); } template ArrayRef detail::LinalgOpInterfaceTraits::ExternalModel::getShape(::mlir::Operation *tablegen_opaque_val, OpOperand*opOperand) const { Type t = opOperand->get().getType(); // A VectorType is an elemental type, do not consider its rank for the operand. if (isa(t)) return {}; if (auto shapedType = ::llvm::dyn_cast(t)) { // Failsafe. assert((isa(t) || isa(t)) && "expected a ranked tensor or memref in LinalgInterface::getRank"); return shapedType.getShape(); } return {}; } } // namespace mlir::linalg namespace mlir::linalg { template FailureOr> detail::AggregatedOpInterfaceInterfaceTraits::Model::decomposeOperation(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpBuilder & b) { return (llvm::cast(tablegen_opaque_val)).decomposeOperation(b); } template FailureOr> detail::AggregatedOpInterfaceInterfaceTraits::FallbackModel::decomposeOperation(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, OpBuilder & b) { return static_cast(impl)->decomposeOperation(tablegen_opaque_val, b); } template FailureOr> detail::AggregatedOpInterfaceInterfaceTraits::ExternalModel::decomposeOperation(::mlir::Operation *tablegen_opaque_val, OpBuilder &b) const { return {}; } } // namespace mlir::linalg