//===----------------------------------------------------------------------===// // Implementation of CopyOp //===----------------------------------------------------------------------===// SmallVector CopyOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr CopyOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned CopyOp::getNumRegionArgs() { return 2; } std::string CopyOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void CopyOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "CopyOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; TypeFn castVal = TypeFn::cast_signed; auto castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) { return attr.getName() == "cast"; }); if (castIter != attrs.end()) { if (auto attr = llvm::dyn_cast(castIter->getValue())) castVal = attr.getValue(); } Value value1 = helper.buildTypeFn(castVal, block.getArgument(1).getType(), block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult CopyOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, CopyOp::getNumRegionArgs(), CopyOp::getRegionBuilder()); } void CopyOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult CopyOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void CopyOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability CopyOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of ExpOp //===----------------------------------------------------------------------===// SmallVector ExpOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr ExpOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned ExpOp::getNumRegionArgs() { return 2; } std::string ExpOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void ExpOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "ExpOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::exp, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult ExpOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, ExpOp::getNumRegionArgs(), ExpOp::getRegionBuilder()); } void ExpOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult ExpOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void ExpOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability ExpOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of LogOp //===----------------------------------------------------------------------===// SmallVector LogOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr LogOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned LogOp::getNumRegionArgs() { return 2; } std::string LogOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void LogOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "LogOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::log, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult LogOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, LogOp::getNumRegionArgs(), LogOp::getRegionBuilder()); } void LogOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult LogOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void LogOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability LogOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of AbsOp //===----------------------------------------------------------------------===// SmallVector AbsOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr AbsOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned AbsOp::getNumRegionArgs() { return 2; } std::string AbsOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void AbsOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "AbsOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::abs, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult AbsOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, AbsOp::getNumRegionArgs(), AbsOp::getRegionBuilder()); } void AbsOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult AbsOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void AbsOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability AbsOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of CeilOp //===----------------------------------------------------------------------===// SmallVector CeilOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr CeilOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned CeilOp::getNumRegionArgs() { return 2; } std::string CeilOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void CeilOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "CeilOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::ceil, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult CeilOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, CeilOp::getNumRegionArgs(), CeilOp::getRegionBuilder()); } void CeilOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult CeilOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void CeilOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability CeilOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of FloorOp //===----------------------------------------------------------------------===// SmallVector FloorOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr FloorOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned FloorOp::getNumRegionArgs() { return 2; } std::string FloorOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void FloorOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "FloorOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::floor, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult FloorOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, FloorOp::getNumRegionArgs(), FloorOp::getRegionBuilder()); } void FloorOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult FloorOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void FloorOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability FloorOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of NegFOp //===----------------------------------------------------------------------===// SmallVector NegFOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr NegFOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned NegFOp::getNumRegionArgs() { return 2; } std::string NegFOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void NegFOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "NegFOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::negf, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult NegFOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, NegFOp::getNumRegionArgs(), NegFOp::getRegionBuilder()); } void NegFOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult NegFOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void NegFOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability NegFOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of ReciprocalOp //===----------------------------------------------------------------------===// SmallVector ReciprocalOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr ReciprocalOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned ReciprocalOp::getNumRegionArgs() { return 2; } std::string ReciprocalOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void ReciprocalOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "ReciprocalOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::reciprocal, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult ReciprocalOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, ReciprocalOp::getNumRegionArgs(), ReciprocalOp::getRegionBuilder()); } void ReciprocalOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult ReciprocalOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void ReciprocalOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability ReciprocalOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of RoundOp //===----------------------------------------------------------------------===// SmallVector RoundOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr RoundOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned RoundOp::getNumRegionArgs() { return 2; } std::string RoundOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void RoundOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "RoundOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::round, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult RoundOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, RoundOp::getNumRegionArgs(), RoundOp::getRegionBuilder()); } void RoundOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult RoundOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void RoundOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability RoundOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of SqrtOp //===----------------------------------------------------------------------===// SmallVector SqrtOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr SqrtOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned SqrtOp::getNumRegionArgs() { return 2; } std::string SqrtOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void SqrtOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "SqrtOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::sqrt, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult SqrtOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, SqrtOp::getNumRegionArgs(), SqrtOp::getRegionBuilder()); } void SqrtOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult SqrtOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void SqrtOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability SqrtOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of RsqrtOp //===----------------------------------------------------------------------===// SmallVector RsqrtOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr RsqrtOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned RsqrtOp::getNumRegionArgs() { return 2; } std::string RsqrtOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void RsqrtOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "RsqrtOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::rsqrt, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult RsqrtOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, RsqrtOp::getNumRegionArgs(), RsqrtOp::getRegionBuilder()); } void RsqrtOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult RsqrtOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void RsqrtOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability RsqrtOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of SquareOp //===----------------------------------------------------------------------===// SmallVector SquareOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr SquareOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned SquareOp::getNumRegionArgs() { return 2; } std::string SquareOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void SquareOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "SquareOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::square, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult SquareOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, SquareOp::getNumRegionArgs(), SquareOp::getRegionBuilder()); } void SquareOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult SquareOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void SquareOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability SquareOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of TanhOp //===----------------------------------------------------------------------===// SmallVector TanhOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr TanhOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned TanhOp::getNumRegionArgs() { return 2; } std::string TanhOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void TanhOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "TanhOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::tanh, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult TanhOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, TanhOp::getNumRegionArgs(), TanhOp::getRegionBuilder()); } void TanhOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult TanhOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void TanhOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability TanhOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of ErfOp //===----------------------------------------------------------------------===// SmallVector ErfOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr ErfOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned ErfOp::getNumRegionArgs() { return 2; } std::string ErfOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void ErfOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "ErfOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildUnaryFn(UnaryFn::erf, block.getArgument(0), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult ErfOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, ErfOp::getNumRegionArgs(), ErfOp::getRegionBuilder()); } void ErfOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult ErfOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void ErfOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability ErfOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of AddOp //===----------------------------------------------------------------------===// SmallVector AddOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr AddOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned AddOp::getNumRegionArgs() { return 3; } std::string AddOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void AddOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "AddOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult AddOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, AddOp::getNumRegionArgs(), AddOp::getRegionBuilder()); } void AddOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult AddOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void AddOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability AddOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of SubOp //===----------------------------------------------------------------------===// SmallVector SubOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr SubOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned SubOp::getNumRegionArgs() { return 3; } std::string SubOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void SubOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "SubOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::sub, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult SubOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, SubOp::getNumRegionArgs(), SubOp::getRegionBuilder()); } void SubOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult SubOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void SubOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability SubOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of MulOp //===----------------------------------------------------------------------===// SmallVector MulOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr MulOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned MulOp::getNumRegionArgs() { return 3; } std::string MulOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void MulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "MulOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::mul, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult MulOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, MulOp::getNumRegionArgs(), MulOp::getRegionBuilder()); } void MulOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult MulOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void MulOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability MulOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DivOp //===----------------------------------------------------------------------===// SmallVector DivOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr DivOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned DivOp::getNumRegionArgs() { return 3; } std::string DivOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void DivOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DivOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::div, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult DivOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DivOp::getNumRegionArgs(), DivOp::getRegionBuilder()); } void DivOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DivOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DivOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DivOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DivUnsignedOp //===----------------------------------------------------------------------===// SmallVector DivUnsignedOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr DivUnsignedOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned DivUnsignedOp::getNumRegionArgs() { return 3; } std::string DivUnsignedOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void DivUnsignedOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DivUnsignedOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::div_unsigned, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult DivUnsignedOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DivUnsignedOp::getNumRegionArgs(), DivUnsignedOp::getRegionBuilder()); } void DivUnsignedOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DivUnsignedOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DivUnsignedOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DivUnsignedOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of MaxOp //===----------------------------------------------------------------------===// SmallVector MaxOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr MaxOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned MaxOp::getNumRegionArgs() { return 3; } std::string MaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void MaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "MaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult MaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, MaxOp::getNumRegionArgs(), MaxOp::getRegionBuilder()); } void MaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult MaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void MaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability MaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of MinOp //===----------------------------------------------------------------------===// SmallVector MinOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr MinOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned MinOp::getNumRegionArgs() { return 3; } std::string MinOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void MinOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "MinOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::min_signed, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult MinOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, MinOp::getNumRegionArgs(), MinOp::getRegionBuilder()); } void MinOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult MinOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void MinOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability MinOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PowFOp //===----------------------------------------------------------------------===// SmallVector PowFOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr PowFOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned PowFOp::getNumRegionArgs() { return 3; } std::string PowFOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void PowFOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PowFOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildBinaryFn(BinaryFn::powf, block.getArgument(0), block.getArgument(1), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult PowFOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PowFOp::getNumRegionArgs(), PowFOp::getRegionBuilder()); } void PowFOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PowFOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PowFOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PowFOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of SelectOp //===----------------------------------------------------------------------===// SmallVector SelectOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr SelectOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned SelectOp::getNumRegionArgs() { return 4; } std::string SelectOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void SelectOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(4 > 0 && block.getNumArguments() == 4 && "SelectOp regionBuilder expects 4 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTernaryFn(TernaryFn::select, block.getArgument(0), block.getArgument(1), block.getArgument(2), emitError); if (!value1) return; yields.push_back(value1); helper.yieldOutputs(yields); } ParseResult SelectOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, SelectOp::getNumRegionArgs(), SelectOp::getRegionBuilder()); } void SelectOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult SelectOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void SelectOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability SelectOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of QuantizedMatmulOp //===----------------------------------------------------------------------===// SmallVector QuantizedMatmulOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(QuantizedMatmulOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); return exprs; } ArrayAttr QuantizedMatmulOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d2, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned QuantizedMatmulOp::getNumRegionArgs() { return 5; } std::string QuantizedMatmulOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void QuantizedMatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "QuantizedMatmulOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult QuantizedMatmulOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, QuantizedMatmulOp::getNumRegionArgs(), QuantizedMatmulOp::getRegionBuilder()); } void QuantizedMatmulOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult QuantizedMatmulOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void QuantizedMatmulOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability QuantizedMatmulOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Mmt4DOp //===----------------------------------------------------------------------===// SmallVector Mmt4DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Mmt4DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr Mmt4DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d0, d2, d3, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d1, d2, d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Mmt4DOp::getNumRegionArgs() { return 3; } std::string Mmt4DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void Mmt4DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Mmt4DOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Mmt4DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Mmt4DOp::getNumRegionArgs(), Mmt4DOp::getRegionBuilder()); } void Mmt4DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Mmt4DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Mmt4DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Mmt4DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of BatchMmt4DOp //===----------------------------------------------------------------------===// SmallVector BatchMmt4DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(BatchMmt4DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); return exprs; } ArrayAttr BatchMmt4DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1, d3, d4, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d2, d3, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1, d2, d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned BatchMmt4DOp::getNumRegionArgs() { return 3; } std::string BatchMmt4DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void BatchMmt4DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "BatchMmt4DOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult BatchMmt4DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, BatchMmt4DOp::getNumRegionArgs(), BatchMmt4DOp::getRegionBuilder()); } void BatchMmt4DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult BatchMmt4DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void BatchMmt4DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability BatchMmt4DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of QuantizedBatchMatmulOp //===----------------------------------------------------------------------===// SmallVector QuantizedBatchMatmulOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(QuantizedBatchMatmulOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); return exprs; } ArrayAttr QuantizedBatchMatmulOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d3, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned QuantizedBatchMatmulOp::getNumRegionArgs() { return 5; } std::string QuantizedBatchMatmulOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void QuantizedBatchMatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "QuantizedBatchMatmulOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult QuantizedBatchMatmulOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, QuantizedBatchMatmulOp::getNumRegionArgs(), QuantizedBatchMatmulOp::getRegionBuilder()); } void QuantizedBatchMatmulOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult QuantizedBatchMatmulOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void QuantizedBatchMatmulOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability QuantizedBatchMatmulOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of MatvecOp //===----------------------------------------------------------------------===// SmallVector MatvecOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(MatvecOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); return exprs; } ArrayAttr MatvecOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned MatvecOp::getNumRegionArgs() { return 3; } std::string MatvecOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void MatvecOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "MatvecOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult MatvecOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, MatvecOp::getNumRegionArgs(), MatvecOp::getRegionBuilder()); } void MatvecOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult MatvecOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void MatvecOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability MatvecOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of VecmatOp //===----------------------------------------------------------------------===// SmallVector VecmatOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(VecmatOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); return exprs; } ArrayAttr VecmatOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d1, d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned VecmatOp::getNumRegionArgs() { return 3; } std::string VecmatOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void VecmatOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "VecmatOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult VecmatOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, VecmatOp::getNumRegionArgs(), VecmatOp::getRegionBuilder()); } void VecmatOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult VecmatOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void VecmatOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability VecmatOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of BatchMatvecOp //===----------------------------------------------------------------------===// SmallVector BatchMatvecOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(BatchMatvecOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); return exprs; } ArrayAttr BatchMatvecOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned BatchMatvecOp::getNumRegionArgs() { return 3; } std::string BatchMatvecOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void BatchMatvecOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "BatchMatvecOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult BatchMatvecOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, BatchMatvecOp::getNumRegionArgs(), BatchMatvecOp::getRegionBuilder()); } void BatchMatvecOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult BatchMatvecOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void BatchMatvecOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability BatchMatvecOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of BatchVecmatOp //===----------------------------------------------------------------------===// SmallVector BatchVecmatOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(BatchVecmatOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); return exprs; } ArrayAttr BatchVecmatOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 3, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned BatchVecmatOp::getNumRegionArgs() { return 3; } std::string BatchVecmatOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void BatchVecmatOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "BatchVecmatOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult BatchVecmatOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, BatchVecmatOp::getNumRegionArgs(), BatchVecmatOp::getRegionBuilder()); } void BatchVecmatOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult BatchVecmatOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void BatchVecmatOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability BatchVecmatOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DotOp //===----------------------------------------------------------------------===// SmallVector DotOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DotOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); return exprs; } ArrayAttr DotOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0)[s0] -> (d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 1, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0)[s0] -> (d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 1, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0)[s0] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 1, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DotOp::getNumRegionArgs() { return 3; } std::string DotOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void DotOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DotOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DotOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DotOp::getNumRegionArgs(), DotOp::getRegionBuilder()); } void DotOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DotOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DotOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DotOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv1DOp //===----------------------------------------------------------------------===// SmallVector Conv1DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv1DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); return exprs; } ArrayAttr Conv1DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0 + d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv1DOp::getNumRegionArgs() { return 3; } std::string Conv1DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void Conv1DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv1DOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv1DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv1DOp::getNumRegionArgs(), Conv1DOp::getRegionBuilder()); } void Conv1DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv1DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv1DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv1DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DOp //===----------------------------------------------------------------------===// SmallVector Conv2DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); return exprs; } ArrayAttr Conv2DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0 + d2, d1 + d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DOp::getNumRegionArgs() { return 3; } std::string Conv2DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void Conv2DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DOp::getNumRegionArgs(), Conv2DOp::getRegionBuilder()); } void Conv2DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv3DOp //===----------------------------------------------------------------------===// SmallVector Conv3DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv3DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr Conv3DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d0 + d3, d1 + d4, d2 + d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d3, d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv3DOp::getNumRegionArgs() { return 3; } std::string Conv3DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void Conv3DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv3DOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv3DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv3DOp::getNumRegionArgs(), Conv3DOp::getRegionBuilder()); } void Conv3DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv3DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv3DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv3DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv1DNwcWcfOp //===----------------------------------------------------------------------===// SmallVector Conv1DNwcWcfOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv1DNwcWcfOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); return exprs; } ArrayAttr Conv1DNwcWcfOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1 * s2 + d3 * s4, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d3, d4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv1DNwcWcfOp::getNumRegionArgs() { return 3; } std::string Conv1DNwcWcfOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv1DNwcWcfOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv1DNwcWcfOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv1DNwcWcfOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv1DNwcWcfOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv1DNwcWcfOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv1DNwcWcfOp::getNumRegionArgs(), Conv1DNwcWcfOp::getRegionBuilder()); } void Conv1DNwcWcfOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv1DNwcWcfOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv1DNwcWcfOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv1DNwcWcfOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv1DNcwFcwOp //===----------------------------------------------------------------------===// SmallVector Conv1DNcwFcwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv1DNcwFcwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); return exprs; } ArrayAttr Conv1DNcwFcwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d3, d2 * s3 + d4 * s5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d1, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv1DNcwFcwOp::getNumRegionArgs() { return 3; } std::string Conv1DNcwFcwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv1DNcwFcwOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv1DNcwFcwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv1DNcwFcwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv1DNcwFcwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv1DNcwFcwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv1DNcwFcwOp::getNumRegionArgs(), Conv1DNcwFcwOp::getRegionBuilder()); } void Conv1DNcwFcwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv1DNcwFcwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv1DNcwFcwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv1DNcwFcwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwcHwcfOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwcHwcfOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwcHwcfOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNhwcHwcfOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d4, d5, d6, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwcHwcfOp::getNumRegionArgs() { return 3; } std::string Conv2DNhwcHwcfOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwcHwcfOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwcHwcfOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwcHwcfOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNhwcHwcfOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNhwcHwcfOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwcHwcfOp::getNumRegionArgs(), Conv2DNhwcHwcfOp::getRegionBuilder()); } void Conv2DNhwcHwcfOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwcHwcfOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwcHwcfOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwcHwcfOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwcFhwcOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwcFhwcOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwcFhwcOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNhwcFhwcOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d3, d4, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwcFhwcOp::getNumRegionArgs() { return 3; } std::string Conv2DNhwcFhwcOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwcFhwcOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwcFhwcOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwcFhwcOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNhwcFhwcOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNhwcFhwcOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwcFhwcOp::getNumRegionArgs(), Conv2DNhwcFhwcOp::getRegionBuilder()); } void Conv2DNhwcFhwcOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwcFhwcOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwcFhwcOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwcFhwcOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwcHwcfQOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwcHwcfQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwcHwcfQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNhwcHwcfQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d4, d5, d6, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwcHwcfQOp::getNumRegionArgs() { return 5; } std::string Conv2DNhwcHwcfQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwcHwcfQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwcHwcfQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwcHwcfQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv2DNhwcHwcfQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv2DNhwcHwcfQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwcHwcfQOp::getNumRegionArgs(), Conv2DNhwcHwcfQOp::getRegionBuilder()); } void Conv2DNhwcHwcfQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwcHwcfQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwcHwcfQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwcHwcfQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwcFhwcQOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwcFhwcQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwcFhwcQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNhwcFhwcQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d3, d4, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwcFhwcQOp::getNumRegionArgs() { return 5; } std::string Conv2DNhwcFhwcQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwcFhwcQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwcFhwcQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwcFhwcQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv2DNhwcFhwcQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv2DNhwcFhwcQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwcFhwcQOp::getNumRegionArgs(), Conv2DNhwcFhwcQOp::getRegionBuilder()); } void Conv2DNhwcFhwcQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwcFhwcQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwcFhwcQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwcFhwcQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNchwFchwQOp //===----------------------------------------------------------------------===// SmallVector Conv2DNchwFchwQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNchwFchwQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNchwFchwQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d4, d2 * s3 + d5 * s5, d3 * s7 + d6 * s9)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d1, d4, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNchwFchwQOp::getNumRegionArgs() { return 5; } std::string Conv2DNchwFchwQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNchwFchwQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNchwFchwQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNchwFchwQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv2DNchwFchwQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv2DNchwFchwQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNchwFchwQOp::getNumRegionArgs(), Conv2DNchwFchwQOp::getRegionBuilder()); } void Conv2DNchwFchwQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNchwFchwQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNchwFchwQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNchwFchwQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNchwFchwOp //===----------------------------------------------------------------------===// SmallVector Conv2DNchwFchwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNchwFchwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr Conv2DNchwFchwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d4, d2 * s3 + d5 * s5, d3 * s7 + d6 * s9)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d1, d4, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNchwFchwOp::getNumRegionArgs() { return 3; } std::string Conv2DNchwFchwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNchwFchwOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNchwFchwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNchwFchwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNchwFchwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNchwFchwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNchwFchwOp::getNumRegionArgs(), Conv2DNchwFchwOp::getRegionBuilder()); } void Conv2DNchwFchwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNchwFchwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNchwFchwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNchwFchwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNgchwFgchwOp //===----------------------------------------------------------------------===// SmallVector Conv2DNgchwFgchwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNgchwFgchwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); return exprs; } ArrayAttr Conv2DNgchwFgchwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d5, d3 * s4 + d6 * s6, d4 * s8 + d7 * s10)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d2, d1, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNgchwFgchwOp::getNumRegionArgs() { return 3; } std::string Conv2DNgchwFgchwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNgchwFgchwOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNgchwFgchwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNgchwFgchwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNgchwFgchwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNgchwFgchwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNgchwFgchwOp::getNumRegionArgs(), Conv2DNgchwFgchwOp::getRegionBuilder()); } void Conv2DNgchwFgchwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNgchwFgchwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNgchwFgchwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNgchwFgchwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNgchwGfchwOp //===----------------------------------------------------------------------===// SmallVector Conv2DNgchwGfchwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNgchwGfchwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); return exprs; } ArrayAttr Conv2DNgchwGfchwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d5, d3 * s4 + d6 * s6, d4 * s8 + d7 * s10)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d1, d2, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNgchwGfchwOp::getNumRegionArgs() { return 3; } std::string Conv2DNgchwGfchwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNgchwGfchwOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNgchwGfchwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNgchwGfchwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNgchwGfchwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNgchwGfchwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNgchwGfchwOp::getNumRegionArgs(), Conv2DNgchwGfchwOp::getRegionBuilder()); } void Conv2DNgchwGfchwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNgchwGfchwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNgchwGfchwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNgchwGfchwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwgcGfhwcOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwgcGfhwcOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwgcGfhwcOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); return exprs; } ArrayAttr Conv2DNhwgcGfhwcOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d3, d4, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwgcGfhwcOp::getNumRegionArgs() { return 3; } std::string Conv2DNhwgcGfhwcOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwgcGfhwcOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwgcGfhwcOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwgcGfhwcOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv2DNhwgcGfhwcOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv2DNhwgcGfhwcOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwgcGfhwcOp::getNumRegionArgs(), Conv2DNhwgcGfhwcOp::getRegionBuilder()); } void Conv2DNhwgcGfhwcOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwgcGfhwcOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwgcGfhwcOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwgcGfhwcOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNhwgcGfhwcQOp //===----------------------------------------------------------------------===// SmallVector Conv2DNhwgcGfhwcQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNhwgcGfhwcQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); return exprs; } ArrayAttr Conv2DNhwgcGfhwcQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d3, d4, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNhwgcGfhwcQOp::getNumRegionArgs() { return 5; } std::string Conv2DNhwgcGfhwcQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNhwgcGfhwcQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNhwgcGfhwcQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNhwgcGfhwcQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv2DNhwgcGfhwcQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv2DNhwgcGfhwcQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNhwgcGfhwcQOp::getNumRegionArgs(), Conv2DNhwgcGfhwcQOp::getRegionBuilder()); } void Conv2DNhwgcGfhwcQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNhwgcGfhwcQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNhwgcGfhwcQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNhwgcGfhwcQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv2DNgchwGfchwQOp //===----------------------------------------------------------------------===// SmallVector Conv2DNgchwGfchwQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv2DNgchwGfchwQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); return exprs; } ArrayAttr Conv2DNgchwGfchwQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d5, d3 * s4 + d6 * s6, d4 * s8 + d7 * s10)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d1, d2, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv2DNgchwGfchwQOp::getNumRegionArgs() { return 5; } std::string Conv2DNgchwGfchwQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv2DNgchwGfchwQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv2DNgchwGfchwQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv2DNgchwGfchwQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv2DNgchwGfchwQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv2DNgchwGfchwQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv2DNgchwGfchwQOp::getNumRegionArgs(), Conv2DNgchwGfchwQOp::getRegionBuilder()); } void Conv2DNgchwGfchwQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv2DNgchwGfchwQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv2DNgchwGfchwQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv2DNgchwGfchwQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv3DNdhwcDhwcfOp //===----------------------------------------------------------------------===// SmallVector Conv3DNdhwcDhwcfOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv3DNdhwcDhwcfOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); exprs.push_back(getAffineSymbolExpr(14, context)); return exprs; } ArrayAttr Conv3DNdhwcDhwcfOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d8)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d5, d6, d7, d8, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv3DNdhwcDhwcfOp::getNumRegionArgs() { return 3; } std::string Conv3DNdhwcDhwcfOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv3DNdhwcDhwcfOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv3DNdhwcDhwcfOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv3DNdhwcDhwcfOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv3DNdhwcDhwcfOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv3DNdhwcDhwcfOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv3DNdhwcDhwcfOp::getNumRegionArgs(), Conv3DNdhwcDhwcfOp::getRegionBuilder()); } void Conv3DNdhwcDhwcfOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv3DNdhwcDhwcfOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv3DNdhwcDhwcfOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv3DNdhwcDhwcfOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv3DNdhwcDhwcfQOp //===----------------------------------------------------------------------===// SmallVector Conv3DNdhwcDhwcfQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv3DNdhwcDhwcfQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); exprs.push_back(getAffineSymbolExpr(14, context)); return exprs; } ArrayAttr Conv3DNdhwcDhwcfQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d8)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d5, d6, d7, d8, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv3DNdhwcDhwcfQOp::getNumRegionArgs() { return 5; } std::string Conv3DNdhwcDhwcfQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv3DNdhwcDhwcfQOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv3DNdhwcDhwcfQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv3DNdhwcDhwcfQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "Conv3DNdhwcDhwcfQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult Conv3DNdhwcDhwcfQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv3DNdhwcDhwcfQOp::getNumRegionArgs(), Conv3DNdhwcDhwcfQOp::getRegionBuilder()); } void Conv3DNdhwcDhwcfQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv3DNdhwcDhwcfQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv3DNdhwcDhwcfQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv3DNdhwcDhwcfQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of Conv3DNcdhwFcdhwOp //===----------------------------------------------------------------------===// SmallVector Conv3DNcdhwFcdhwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(Conv3DNcdhwFcdhwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); int64_t cst11 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst11, context)); exprs.push_back(getAffineSymbolExpr(12, context)); int64_t cst13 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst13, context)); exprs.push_back(getAffineSymbolExpr(14, context)); return exprs; } ArrayAttr Conv3DNcdhwFcdhwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d5, d2 * s3 + d6 * s5, d3 * s7 + d7 * s9, d4 * s11 + d8 * s13)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d1, d5, d6, d7, d8)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned Conv3DNcdhwFcdhwOp::getNumRegionArgs() { return 3; } std::string Conv3DNcdhwFcdhwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool Conv3DNcdhwFcdhwOp::hasDynamicIndexingMaps() { return true; } LogicalResult Conv3DNcdhwFcdhwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void Conv3DNcdhwFcdhwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "Conv3DNcdhwFcdhwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult Conv3DNcdhwFcdhwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, Conv3DNcdhwFcdhwOp::getNumRegionArgs(), Conv3DNcdhwFcdhwOp::getRegionBuilder()); } void Conv3DNcdhwFcdhwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult Conv3DNcdhwFcdhwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void Conv3DNcdhwFcdhwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability Conv3DNcdhwFcdhwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv1DNwcWcOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv1DNwcWcOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv1DNwcWcOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr DepthwiseConv1DNwcWcOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv1DNwcWcOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv1DNwcWcOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv1DNwcWcOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv1DNwcWcOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv1DNwcWcOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv1DNwcWcOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv1DNwcWcOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv1DNwcWcOp::getNumRegionArgs(), DepthwiseConv1DNwcWcOp::getRegionBuilder()); } void DepthwiseConv1DNwcWcOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv1DNwcWcOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv1DNwcWcOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv1DNwcWcOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv1DNcwCwOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv1DNcwCwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv1DNcwCwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); return exprs; } ArrayAttr DepthwiseConv1DNcwCwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d2, d1 * s3 + d3 * s5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d2, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv1DNcwCwOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv1DNcwCwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv1DNcwCwOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv1DNcwCwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv1DNcwCwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv1DNcwCwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv1DNcwCwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv1DNcwCwOp::getNumRegionArgs(), DepthwiseConv1DNcwCwOp::getRegionBuilder()); } void DepthwiseConv1DNcwCwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv1DNcwCwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv1DNcwCwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv1DNcwCwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv1DNwcWcmOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv1DNwcWcmOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv1DNwcWcmOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); return exprs; } ArrayAttr DepthwiseConv1DNwcWcmOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1 * s2 + d4 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d4, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4)[s0, s1, s2, s3, s4, s5, s6] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 5, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv1DNwcWcmOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv1DNwcWcmOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv1DNwcWcmOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv1DNwcWcmOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv1DNwcWcmOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv1DNwcWcmOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv1DNwcWcmOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv1DNwcWcmOp::getNumRegionArgs(), DepthwiseConv1DNwcWcmOp::getRegionBuilder()); } void DepthwiseConv1DNwcWcmOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv1DNwcWcmOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv1DNwcWcmOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv1DNwcWcmOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv2DNhwcHwcOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv2DNhwcHwcOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv2DNhwcHwcOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr DepthwiseConv2DNhwcHwcOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv2DNhwcHwcOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv2DNhwcHwcOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv2DNhwcHwcOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv2DNhwcHwcOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv2DNhwcHwcOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv2DNhwcHwcOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv2DNhwcHwcOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv2DNhwcHwcOp::getNumRegionArgs(), DepthwiseConv2DNhwcHwcOp::getRegionBuilder()); } void DepthwiseConv2DNhwcHwcOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv2DNhwcHwcOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv2DNhwcHwcOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv2DNhwcHwcOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv2DNchwChwOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv2DNchwChwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv2DNchwChwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); return exprs; } ArrayAttr DepthwiseConv2DNchwChwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d3, d1 * s3 + d4 * s5, d2 * s7 + d5 * s9)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d3, d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d3, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv2DNchwChwOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv2DNchwChwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv2DNchwChwOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv2DNchwChwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv2DNchwChwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv2DNchwChwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv2DNchwChwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv2DNchwChwOp::getNumRegionArgs(), DepthwiseConv2DNchwChwOp::getRegionBuilder()); } void DepthwiseConv2DNchwChwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv2DNchwChwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv2DNchwChwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv2DNchwChwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv2DNhwcHwcQOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv2DNhwcHwcQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv2DNhwcHwcQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr DepthwiseConv2DNhwcHwcQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv2DNhwcHwcQOp::getNumRegionArgs() { return 5; } std::string DepthwiseConv2DNhwcHwcQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv2DNhwcHwcQOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv2DNhwcHwcQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv2DNhwcHwcQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "DepthwiseConv2DNhwcHwcQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult DepthwiseConv2DNhwcHwcQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv2DNhwcHwcQOp::getNumRegionArgs(), DepthwiseConv2DNhwcHwcQOp::getRegionBuilder()); } void DepthwiseConv2DNhwcHwcQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv2DNhwcHwcQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv2DNhwcHwcQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv2DNhwcHwcQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv2DNhwcHwcmOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv2DNhwcHwcmOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv2DNhwcHwcmOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr DepthwiseConv2DNhwcHwcmOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d5, d6, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv2DNhwcHwcmOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv2DNhwcHwcmOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv2DNhwcHwcmOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv2DNhwcHwcmOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv2DNhwcHwcmOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv2DNhwcHwcmOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv2DNhwcHwcmOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv2DNhwcHwcmOp::getNumRegionArgs(), DepthwiseConv2DNhwcHwcmOp::getRegionBuilder()); } void DepthwiseConv2DNhwcHwcmOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv2DNhwcHwcmOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv2DNhwcHwcmOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv2DNhwcHwcmOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv2DNhwcHwcmQOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv2DNhwcHwcmQOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(DepthwiseConv2DNhwcHwcmQOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); return exprs; } ArrayAttr DepthwiseConv2DNhwcHwcmQOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d5, d6, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 7, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv2DNhwcHwcmQOp::getNumRegionArgs() { return 5; } std::string DepthwiseConv2DNhwcHwcmQOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv2DNhwcHwcmQOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv2DNhwcHwcmQOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv2DNhwcHwcmQOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(5 > 0 && block.getNumArguments() == 5 && "DepthwiseConv2DNhwcHwcmQOp regionBuilder expects 5 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(2), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::sub, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(1), emitError); if (!value4) return; Value value5 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(4).getType(), block.getArgument(3), emitError); if (!value5) return; Value value6 = helper.buildBinaryFn(BinaryFn::sub, value4, value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::mul, value3, value6, emitError); if (!value7) return; Value value8 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(4), value7, emitError); if (!value8) return; yields.push_back(value8); helper.yieldOutputs(yields); } ParseResult DepthwiseConv2DNhwcHwcmQOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv2DNhwcHwcmQOp::getNumRegionArgs(), DepthwiseConv2DNhwcHwcmQOp::getRegionBuilder()); } void DepthwiseConv2DNhwcHwcmQOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv2DNhwcHwcmQOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv2DNhwcHwcmQOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv2DNhwcHwcmQOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv3DNdhwcDhwcOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv3DNdhwcDhwcOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::parallel }; } static SmallVector getSymbolBindings(DepthwiseConv3DNdhwcDhwcOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); return exprs; } ArrayAttr DepthwiseConv3DNdhwcDhwcOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3 * s10 + d6 * s12, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d4, d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1, d2, d3, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv3DNdhwcDhwcOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv3DNdhwcDhwcOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv3DNdhwcDhwcOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv3DNdhwcDhwcOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv3DNdhwcDhwcOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv3DNdhwcDhwcOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv3DNdhwcDhwcOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv3DNdhwcDhwcOp::getNumRegionArgs(), DepthwiseConv3DNdhwcDhwcOp::getRegionBuilder()); } void DepthwiseConv3DNdhwcDhwcOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv3DNdhwcDhwcOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv3DNdhwcDhwcOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv3DNdhwcDhwcOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv3DNcdhwCdhwOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv3DNcdhwCdhwOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::parallel }; } static SmallVector getSymbolBindings(DepthwiseConv3DNcdhwCdhwOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); exprs.push_back(getAffineSymbolExpr(10, context)); int64_t cst11 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst11, context)); exprs.push_back(getAffineSymbolExpr(12, context)); int64_t cst13 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst13, context)); return exprs; } ArrayAttr DepthwiseConv3DNcdhwCdhwOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d7, d1 * s3 + d4 * s5, d2 * s7 + d5 * s9, d3 * s11 + d6 * s13)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d7, d4, d5, d6)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d7, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv3DNcdhwCdhwOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv3DNcdhwCdhwOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv3DNcdhwCdhwOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv3DNcdhwCdhwOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv3DNcdhwCdhwOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv3DNcdhwCdhwOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv3DNcdhwCdhwOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv3DNcdhwCdhwOp::getNumRegionArgs(), DepthwiseConv3DNcdhwCdhwOp::getRegionBuilder()); } void DepthwiseConv3DNcdhwCdhwOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv3DNcdhwCdhwOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv3DNcdhwCdhwOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv3DNcdhwCdhwOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of DepthwiseConv3DNdhwcDhwcmOp //===----------------------------------------------------------------------===// SmallVector DepthwiseConv3DNdhwcDhwcmOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::parallel }; } static SmallVector getSymbolBindings(DepthwiseConv3DNdhwcDhwcmOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); exprs.push_back(getAffineSymbolExpr(14, context)); return exprs; } ArrayAttr DepthwiseConv3DNdhwcDhwcmOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d8)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d5, d6, d7, d8, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7, d8)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14] -> (d0, d1, d2, d3, d8, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 9, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned DepthwiseConv3DNdhwcDhwcmOp::getNumRegionArgs() { return 3; } std::string DepthwiseConv3DNdhwcDhwcmOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool DepthwiseConv3DNdhwcDhwcmOp::hasDynamicIndexingMaps() { return true; } LogicalResult DepthwiseConv3DNdhwcDhwcmOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void DepthwiseConv3DNdhwcDhwcmOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "DepthwiseConv3DNdhwcDhwcmOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(1), emitError); if (!value2) return; Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError); if (!value3) return; Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3, emitError); if (!value4) return; yields.push_back(value4); helper.yieldOutputs(yields); } ParseResult DepthwiseConv3DNdhwcDhwcmOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, DepthwiseConv3DNdhwcDhwcmOp::getNumRegionArgs(), DepthwiseConv3DNdhwcDhwcmOp::getRegionBuilder()); } void DepthwiseConv3DNdhwcDhwcmOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult DepthwiseConv3DNdhwcDhwcmOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void DepthwiseConv3DNdhwcDhwcmOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability DepthwiseConv3DNdhwcDhwcmOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNhwcSumOp //===----------------------------------------------------------------------===// SmallVector PoolingNhwcSumOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNhwcSumOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr PoolingNhwcSumOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNhwcSumOp::getNumRegionArgs() { return 3; } std::string PoolingNhwcSumOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNhwcSumOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNhwcSumOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNhwcSumOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNhwcSumOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNhwcSumOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNhwcSumOp::getNumRegionArgs(), PoolingNhwcSumOp::getRegionBuilder()); } void PoolingNhwcSumOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNhwcSumOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNhwcSumOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNhwcSumOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNchwSumOp //===----------------------------------------------------------------------===// SmallVector PoolingNchwSumOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNchwSumOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); return exprs; } ArrayAttr PoolingNchwSumOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2 * s3 + d4 * s5, d3 * s7 + d5 * s9)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNchwSumOp::getNumRegionArgs() { return 3; } std::string PoolingNchwSumOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNchwSumOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNchwSumOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNchwSumOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNchwSumOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNchwSumOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNchwSumOp::getNumRegionArgs(), PoolingNchwSumOp::getRegionBuilder()); } void PoolingNchwSumOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNchwSumOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNchwSumOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNchwSumOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNhwcMaxOp //===----------------------------------------------------------------------===// SmallVector PoolingNhwcMaxOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNhwcMaxOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr PoolingNhwcMaxOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNhwcMaxOp::getNumRegionArgs() { return 3; } std::string PoolingNhwcMaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNhwcMaxOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNhwcMaxOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNhwcMaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNhwcMaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNhwcMaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNhwcMaxOp::getNumRegionArgs(), PoolingNhwcMaxOp::getRegionBuilder()); } void PoolingNhwcMaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNhwcMaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNhwcMaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNhwcMaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNhwcMaxUnsignedOp //===----------------------------------------------------------------------===// SmallVector PoolingNhwcMaxUnsignedOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNhwcMaxUnsignedOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr PoolingNhwcMaxUnsignedOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNhwcMaxUnsignedOp::getNumRegionArgs() { return 3; } std::string PoolingNhwcMaxUnsignedOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNhwcMaxUnsignedOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNhwcMaxUnsignedOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNhwcMaxUnsignedOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNhwcMaxUnsignedOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_unsigned, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_unsigned, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNhwcMaxUnsignedOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNhwcMaxUnsignedOp::getNumRegionArgs(), PoolingNhwcMaxUnsignedOp::getRegionBuilder()); } void PoolingNhwcMaxUnsignedOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNhwcMaxUnsignedOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNhwcMaxUnsignedOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNhwcMaxUnsignedOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNchwMaxOp //===----------------------------------------------------------------------===// SmallVector PoolingNchwMaxOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNchwMaxOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); exprs.push_back(getAffineSymbolExpr(6, context)); int64_t cst7 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst7, context)); exprs.push_back(getAffineSymbolExpr(8, context)); int64_t cst9 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst9, context)); return exprs; } ArrayAttr PoolingNchwMaxOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2 * s3 + d4 * s5, d3 * s7 + d5 * s9)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNchwMaxOp::getNumRegionArgs() { return 3; } std::string PoolingNchwMaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNchwMaxOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNchwMaxOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNchwMaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNchwMaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNchwMaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNchwMaxOp::getNumRegionArgs(), PoolingNchwMaxOp::getRegionBuilder()); } void PoolingNchwMaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNchwMaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNchwMaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNchwMaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNhwcMinOp //===----------------------------------------------------------------------===// SmallVector PoolingNhwcMinOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNhwcMinOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr PoolingNhwcMinOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNhwcMinOp::getNumRegionArgs() { return 3; } std::string PoolingNhwcMinOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNhwcMinOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNhwcMinOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNhwcMinOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNhwcMinOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::min_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNhwcMinOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNhwcMinOp::getNumRegionArgs(), PoolingNhwcMinOp::getRegionBuilder()); } void PoolingNhwcMinOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNhwcMinOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNhwcMinOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNhwcMinOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNhwcMinUnsignedOp //===----------------------------------------------------------------------===// SmallVector PoolingNhwcMinUnsignedOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNhwcMinUnsignedOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); return exprs; } ArrayAttr PoolingNhwcMinUnsignedOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1 * s2 + d4 * s4, d2 * s6 + d5 * s8, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d4, d5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9] -> (d0, d1, d2, d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 6, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNhwcMinUnsignedOp::getNumRegionArgs() { return 3; } std::string PoolingNhwcMinUnsignedOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNhwcMinUnsignedOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNhwcMinUnsignedOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 2 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNhwcMinUnsignedOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNhwcMinUnsignedOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_unsigned, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::min_unsigned, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNhwcMinUnsignedOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNhwcMinUnsignedOp::getNumRegionArgs(), PoolingNhwcMinUnsignedOp::getRegionBuilder()); } void PoolingNhwcMinUnsignedOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNhwcMinUnsignedOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNhwcMinUnsignedOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNhwcMinUnsignedOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNwcSumOp //===----------------------------------------------------------------------===// SmallVector PoolingNwcSumOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNwcSumOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr PoolingNwcSumOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNwcSumOp::getNumRegionArgs() { return 3; } std::string PoolingNwcSumOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNwcSumOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNwcSumOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNwcSumOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNwcSumOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNwcSumOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNwcSumOp::getNumRegionArgs(), PoolingNwcSumOp::getRegionBuilder()); } void PoolingNwcSumOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNwcSumOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNwcSumOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNwcSumOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNcwSumOp //===----------------------------------------------------------------------===// SmallVector PoolingNcwSumOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNcwSumOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); return exprs; } ArrayAttr PoolingNcwSumOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2 * s3 + d3 * s5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNcwSumOp::getNumRegionArgs() { return 3; } std::string PoolingNcwSumOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNcwSumOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNcwSumOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNcwSumOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNcwSumOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNcwSumOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNcwSumOp::getNumRegionArgs(), PoolingNcwSumOp::getRegionBuilder()); } void PoolingNcwSumOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNcwSumOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNcwSumOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNcwSumOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNwcMaxOp //===----------------------------------------------------------------------===// SmallVector PoolingNwcMaxOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNwcMaxOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr PoolingNwcMaxOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNwcMaxOp::getNumRegionArgs() { return 3; } std::string PoolingNwcMaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNwcMaxOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNwcMaxOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNwcMaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNwcMaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNwcMaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNwcMaxOp::getNumRegionArgs(), PoolingNwcMaxOp::getRegionBuilder()); } void PoolingNwcMaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNwcMaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNwcMaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNwcMaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNwcMaxUnsignedOp //===----------------------------------------------------------------------===// SmallVector PoolingNwcMaxUnsignedOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNwcMaxUnsignedOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr PoolingNwcMaxUnsignedOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNwcMaxUnsignedOp::getNumRegionArgs() { return 3; } std::string PoolingNwcMaxUnsignedOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNwcMaxUnsignedOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNwcMaxUnsignedOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNwcMaxUnsignedOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNwcMaxUnsignedOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_unsigned, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_unsigned, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNwcMaxUnsignedOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNwcMaxUnsignedOp::getNumRegionArgs(), PoolingNwcMaxUnsignedOp::getRegionBuilder()); } void PoolingNwcMaxUnsignedOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNwcMaxUnsignedOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNwcMaxUnsignedOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNwcMaxUnsignedOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNcwMaxOp //===----------------------------------------------------------------------===// SmallVector PoolingNcwMaxOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNcwMaxOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); exprs.push_back(getAffineSymbolExpr(2, context)); int64_t cst3 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst3, context)); exprs.push_back(getAffineSymbolExpr(4, context)); int64_t cst5 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst5, context)); return exprs; } ArrayAttr PoolingNcwMaxOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2 * s3 + d3 * s5)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNcwMaxOp::getNumRegionArgs() { return 3; } std::string PoolingNcwMaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNcwMaxOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNcwMaxOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNcwMaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNcwMaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNcwMaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNcwMaxOp::getNumRegionArgs(), PoolingNcwMaxOp::getRegionBuilder()); } void PoolingNcwMaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNcwMaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNcwMaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNcwMaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNwcMinOp //===----------------------------------------------------------------------===// SmallVector PoolingNwcMinOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNwcMinOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr PoolingNwcMinOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNwcMinOp::getNumRegionArgs() { return 3; } std::string PoolingNwcMinOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNwcMinOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNwcMinOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNwcMinOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNwcMinOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::min_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNwcMinOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNwcMinOp::getNumRegionArgs(), PoolingNwcMinOp::getRegionBuilder()); } void PoolingNwcMinOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNwcMinOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNwcMinOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNwcMinOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNwcMinUnsignedOp //===----------------------------------------------------------------------===// SmallVector PoolingNwcMinUnsignedOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNwcMinUnsignedOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); return exprs; } ArrayAttr PoolingNwcMinUnsignedOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1 * s2 + d3 * s4, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d3)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4, s5] -> (d0, d1, d2)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 4, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNwcMinUnsignedOp::getNumRegionArgs() { return 3; } std::string PoolingNwcMinUnsignedOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNwcMinUnsignedOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNwcMinUnsignedOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 1 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNwcMinUnsignedOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNwcMinUnsignedOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_unsigned, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::min_unsigned, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNwcMinUnsignedOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNwcMinUnsignedOp::getNumRegionArgs(), PoolingNwcMinUnsignedOp::getRegionBuilder()); } void PoolingNwcMinUnsignedOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNwcMinUnsignedOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNwcMinUnsignedOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNwcMinUnsignedOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNdhwcSumOp //===----------------------------------------------------------------------===// SmallVector PoolingNdhwcSumOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNdhwcSumOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); return exprs; } ArrayAttr PoolingNdhwcSumOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNdhwcSumOp::getNumRegionArgs() { return 3; } std::string PoolingNdhwcSumOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNdhwcSumOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNdhwcSumOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNdhwcSumOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNdhwcSumOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNdhwcSumOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNdhwcSumOp::getNumRegionArgs(), PoolingNdhwcSumOp::getRegionBuilder()); } void PoolingNdhwcSumOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNdhwcSumOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNdhwcSumOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNdhwcSumOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNdhwcMaxOp //===----------------------------------------------------------------------===// SmallVector PoolingNdhwcMaxOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNdhwcMaxOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); return exprs; } ArrayAttr PoolingNdhwcMaxOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNdhwcMaxOp::getNumRegionArgs() { return 3; } std::string PoolingNdhwcMaxOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNdhwcMaxOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNdhwcMaxOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNdhwcMaxOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNdhwcMaxOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::max_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNdhwcMaxOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNdhwcMaxOp::getNumRegionArgs(), PoolingNdhwcMaxOp::getRegionBuilder()); } void PoolingNdhwcMaxOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNdhwcMaxOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNdhwcMaxOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNdhwcMaxOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of PoolingNdhwcMinOp //===----------------------------------------------------------------------===// SmallVector PoolingNdhwcMinOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::parallel, utils::IteratorType::reduction, utils::IteratorType::reduction, utils::IteratorType::reduction }; } static SmallVector getSymbolBindings(PoolingNdhwcMinOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); int64_t cst2 = self.getStrides().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst2, context)); exprs.push_back(getAffineSymbolExpr(3, context)); int64_t cst4 = self.getDilations().getValues()[0]; exprs.push_back(getAffineConstantExpr(cst4, context)); exprs.push_back(getAffineSymbolExpr(5, context)); int64_t cst6 = self.getStrides().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst6, context)); exprs.push_back(getAffineSymbolExpr(7, context)); int64_t cst8 = self.getDilations().getValues()[1]; exprs.push_back(getAffineConstantExpr(cst8, context)); exprs.push_back(getAffineSymbolExpr(9, context)); int64_t cst10 = self.getStrides().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst10, context)); exprs.push_back(getAffineSymbolExpr(11, context)); int64_t cst12 = self.getDilations().getValues()[2]; exprs.push_back(getAffineConstantExpr(cst12, context)); exprs.push_back(getAffineSymbolExpr(13, context)); return exprs; } ArrayAttr PoolingNdhwcMinOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1 * s2 + d5 * s4, d2 * s6 + d6 * s8, d3 * s10 + d7 * s12, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d5, d6, d7)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1, d2, d3, d4, d5, d6, d7)[s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13] -> (d0, d1, d2, d3, d4)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 8, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned PoolingNdhwcMinOp::getNumRegionArgs() { return 3; } std::string PoolingNdhwcMinOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } bool PoolingNdhwcMinOp::hasDynamicIndexingMaps() { return true; } LogicalResult PoolingNdhwcMinOp::verifyIndexingMapRequiredAttributes() { Operation *op = getOperation(); if (auto attr = op->getAttrOfType("strides")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'strides'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'strides'"); } if (auto attr = op->getAttrOfType("dilations")) { if (!attr.getType().getElementType().isInteger(64)) return op->emitError("incorrect element type for index attribute 'dilations'"); if (attr.getType().getShape() != ArrayRef{ 3 }) return op->emitError("incorrect shape for index attribute 'dilations'"); } return success(); } void PoolingNdhwcMinOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(3 > 0 && block.getNumArguments() == 3 && "PoolingNdhwcMinOp regionBuilder expects 3 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(2).getType(), block.getArgument(0), emitError); if (!value1) return; Value value2 = helper.buildBinaryFn(BinaryFn::min_signed, block.getArgument(2), value1, emitError); if (!value2) return; yields.push_back(value2); helper.yieldOutputs(yields); } ParseResult PoolingNdhwcMinOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, PoolingNdhwcMinOp::getNumRegionArgs(), PoolingNdhwcMinOp::getRegionBuilder()); } void PoolingNdhwcMinOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult PoolingNdhwcMinOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void PoolingNdhwcMinOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability PoolingNdhwcMinOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of FillOp //===----------------------------------------------------------------------===// SmallVector FillOp::getIteratorTypesArray() { int64_t rank = getRank(getDpsInitOperand(0)); return SmallVector(rank, utils::IteratorType::parallel); } ArrayAttr FillOp::getIndexingMaps() { MLIRContext *context = getContext(); AffineMap scalarMap = AffineMap::get(getNumParallelLoops(), 0, context); AffineMap tensorMap = AffineMap::getMultiDimIdentityMap( getNumParallelLoops(), context); SmallVector indexingMaps; for (OpOperand &opOperand : getOperation()->getOpOperands()) indexingMaps.push_back(getRank(&opOperand) == 0 ? scalarMap : tensorMap); return Builder(getContext()).getAffineMapArrayAttr(indexingMaps); } unsigned FillOp::getNumRegionArgs() { return 2; } std::string FillOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void FillOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(2 > 0 && block.getNumArguments() == 2 && "FillOp regionBuilder expects 2 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; yields.push_back(block.getArgument(0)); helper.yieldOutputs(yields); } ParseResult FillOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, FillOp::getNumRegionArgs(), FillOp::getRegionBuilder()); } void FillOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult FillOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void FillOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability FillOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); } //===----------------------------------------------------------------------===// // Implementation of FillRng2DOp //===----------------------------------------------------------------------===// SmallVector FillRng2DOp::getIteratorTypesArray() { return SmallVector{ utils::IteratorType::parallel, utils::IteratorType::parallel }; } static SmallVector getSymbolBindings(FillRng2DOp self) { MLIRContext *context = self.getContext(); SmallVector exprs; exprs.push_back(getAffineSymbolExpr(0, context)); exprs.push_back(getAffineSymbolExpr(1, context)); return exprs; } ArrayAttr FillRng2DOp::getIndexingMaps() { static const char memoizeAttr[] = "linalg.memoized_indexing_maps"; ArrayAttr cached = getOperation()->getAttrOfType(memoizeAttr); if (cached) return cached; MLIRContext *context = getContext(); auto symbolBindings = getSymbolBindings(*this); SmallVector maps; maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> ()>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); maps.push_back(llvm::cast(mlir::parseAttribute("affine_map<(d0, d1)[s0, s1] -> (d0, d1)>", context)).getValue()); maps.back() = simplifyAffineMap(maps.back().replaceDimsAndSymbols({}, symbolBindings, 2, 0)); cached = Builder(context).getAffineMapArrayAttr(maps); getOperation()->setAttr(memoizeAttr, cached); return cached; } unsigned FillRng2DOp::getNumRegionArgs() { return 4; } std::string FillRng2DOp::getLibraryCallName() { return generateLibraryCallName(getOperation()); } void FillRng2DOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef attrs, function_ref emitError) { assert(4 > 0 && block.getNumArguments() == 4 && "FillRng2DOp regionBuilder expects 4 (>=0) args"); RegionBuilderHelper helper(b, block); SmallVector yields; Value value1 = helper.constant("2147483647 : i64"); Value value2 = helper.buildTypeFn(TypeFn::cast_signed, helper.getFloat64Type(), value1, emitError); if (!value2) return; Value value3 = helper.index(1); Value value4 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value3, emitError); if (!value4) return; Value value5 = helper.index(0); Value value6 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value5, emitError); if (!value6) return; Value value7 = helper.buildBinaryFn(BinaryFn::add, value6, block.getArgument(2), emitError); if (!value7) return; Value value8 = helper.constant("1103515245 : i64"); Value value9 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value8, emitError); if (!value9) return; Value value10 = helper.buildBinaryFn(BinaryFn::mul, value7, value9, emitError); if (!value10) return; Value value11 = helper.constant("12345 : i64"); Value value12 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value11, emitError); if (!value12) return; Value value13 = helper.buildBinaryFn(BinaryFn::add, value10, value12, emitError); if (!value13) return; Value value14 = helper.buildBinaryFn(BinaryFn::add, value4, value13, emitError); if (!value14) return; Value value15 = helper.constant("1103515245 : i64"); Value value16 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value15, emitError); if (!value16) return; Value value17 = helper.buildBinaryFn(BinaryFn::mul, value14, value16, emitError); if (!value17) return; Value value18 = helper.constant("12345 : i64"); Value value19 = helper.buildTypeFn(TypeFn::cast_signed, helper.getIntegerType(32), value18, emitError); if (!value19) return; Value value20 = helper.buildBinaryFn(BinaryFn::add, value17, value19, emitError); if (!value20) return; Value value21 = helper.buildTypeFn(TypeFn::cast_signed, helper.getFloat64Type(), value20, emitError); if (!value21) return; Value value22 = helper.buildBinaryFn(BinaryFn::add, value2, value21, emitError); if (!value22) return; Value value23 = helper.buildBinaryFn(BinaryFn::sub, block.getArgument(1), block.getArgument(0), emitError); if (!value23) return; Value value24 = helper.constant("2.3283063999999999E-10 : f64"); Value value25 = helper.buildTypeFn(TypeFn::cast_signed, helper.getFloat64Type(), value24, emitError); if (!value25) return; Value value26 = helper.buildBinaryFn(BinaryFn::mul, value23, value25, emitError); if (!value26) return; Value value27 = helper.buildBinaryFn(BinaryFn::mul, value22, value26, emitError); if (!value27) return; Value value28 = helper.buildBinaryFn(BinaryFn::add, value27, block.getArgument(0), emitError); if (!value28) return; Value value29 = helper.buildTypeFn(TypeFn::cast_signed, block.getArgument(3).getType(), value28, emitError); if (!value29) return; yields.push_back(value29); helper.yieldOutputs(yields); } ParseResult FillRng2DOp::parse(OpAsmParser &parser, OperationState &result) { return ::parseNamedStructuredOp(parser, result, FillRng2DOp::getNumRegionArgs(), FillRng2DOp::getRegionBuilder()); } void FillRng2DOp::print(OpAsmPrinter &p) { SmallVector elidedAttrs = {"operandSegmentSizes", "linalg.memoized_indexing_maps"}; ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(), elidedAttrs); } LogicalResult FillRng2DOp::fold(FoldAdaptor, SmallVectorImpl &) { return memref::foldMemRefCast(*this); } void FillRng2DOp::getEffects(SmallVectorImpl< SideEffects::EffectInstance >&effects) { if (hasPureTensorSemantics()) return; getGenericEffectsImpl(effects, cast(getOperation())); } Speculation::Speculatability FillRng2DOp::getSpeculatability() { return getGenericSpeculatabilityImpl(cast(getOperation())); }