1158 lines
77 KiB
C++
1158 lines
77 KiB
C++
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
|
|
|* *|
|
|
|* Interface Declarations *|
|
|
|* *|
|
|
|* Automatically generated file, do not edit! *|
|
|
|* *|
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
namespace mlir {
|
|
|
|
/// Describes an operation allocating a memory slot that can be promoted into
|
|
/// SSA values.
|
|
class PromotableAllocationOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that can load from memory slots and/or store
|
|
/// to memory slots.
|
|
///
|
|
/// For a memory operation on a slot to be valid, it must strictly operate
|
|
/// within the bounds of the slot.
|
|
///
|
|
/// If the same operation does both loads and stores on the same slot, the
|
|
/// load must semantically happen first.
|
|
class PromotableMemOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that can be transformed or deleted so it no longer
|
|
/// uses a provided value (blocking use), in case this would allow the promotion
|
|
/// of a memory slot.
|
|
class PromotableOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that creates a transparent alias of a memory slot
|
|
/// accessed through one of its operands. Mem2Reg traverses chains of these
|
|
/// aliases to project slot values across them. This allows load and store
|
|
/// operations on the alias to be promoted as if they were directly accessing
|
|
/// the underlying slot.
|
|
///
|
|
/// Since an alias remains a blocking use of the underlying slot pointer, the
|
|
/// operation must also implement either `PromotableOpInterface` or
|
|
/// `PromotableMemOpInterface`. This ensures that mem2reg can remove the alias
|
|
/// after the slot has been promoted.
|
|
class PromotableAliaserInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation for which memory slots can be promoted to SSA values
|
|
/// within the operation's regions.
|
|
class PromotableRegionOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations allocating memory slots of aggregates that can be
|
|
/// destructured into multiple smaller allocations.
|
|
class DestructurableAllocationOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations using memory slots in a safe manner.
|
|
class SafeMemorySlotAccessOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations that can access a sub-element of a destructurable slot.
|
|
class DestructurableAccessorOpInterface;
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation allocating a memory slot that can be promoted into
|
|
/// SSA values.
|
|
namespace detail {
|
|
struct PromotableAllocationOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
::llvm::SmallVector<::mlir::MemorySlot> (*getPromotableSlots)(const Concept *impl, ::mlir::Operation *);
|
|
::mlir::Value (*getDefaultValue)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::OpBuilder &);
|
|
void (*handleBlockArgument)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::BlockArgument, ::mlir::OpBuilder &);
|
|
::std::optional<::mlir::PromotableAllocationOpInterface> (*handlePromotionComplete)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::Value, ::mlir::OpBuilder &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableAllocationOpInterface;
|
|
Model() : Concept{getPromotableSlots, getDefaultValue, handleBlockArgument, handlePromotionComplete} {}
|
|
|
|
static inline ::llvm::SmallVector<::mlir::MemorySlot> getPromotableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline ::mlir::Value getDefaultValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
static inline void handleBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::BlockArgument argument, ::mlir::OpBuilder & builder);
|
|
static inline ::std::optional<::mlir::PromotableAllocationOpInterface> handlePromotionComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value defaultValue, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableAllocationOpInterface;
|
|
FallbackModel() : Concept{getPromotableSlots, getDefaultValue, handleBlockArgument, handlePromotionComplete} {}
|
|
|
|
static inline ::llvm::SmallVector<::mlir::MemorySlot> getPromotableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline ::mlir::Value getDefaultValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
static inline void handleBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::BlockArgument argument, ::mlir::OpBuilder & builder);
|
|
static inline ::std::optional<::mlir::PromotableAllocationOpInterface> handlePromotionComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value defaultValue, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct PromotableAllocationOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class PromotableAllocationOpInterface : public ::mlir::OpInterface<PromotableAllocationOpInterface, detail::PromotableAllocationOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<PromotableAllocationOpInterface, detail::PromotableAllocationOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::PromotableAllocationOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Returns a list of memory slots for which promotion should be attempted.
|
|
/// This only considers the local semantics of the allocator, ignoring
|
|
/// whether the slot pointer is properly used or not. This allocator is the
|
|
/// "owner" of the returned slots, meaning no two allocators should return
|
|
/// the same slot. The content of the memory slot must only be reachable
|
|
/// using loads and stores to the provided slot pointer, no aliasing is
|
|
/// allowed.
|
|
///
|
|
/// Promotion of the slot will lead to the slot pointer no longer being
|
|
/// used, leaving the content of the memory slot unreachable.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
::llvm::SmallVector<::mlir::MemorySlot> getPromotableSlots();
|
|
|
|
/// Provides the default Value of this memory slot. The provided Value
|
|
/// will be used as the reaching definition of loads done before any store.
|
|
/// This Value must outlive the promotion and dominate all the uses of this
|
|
/// slot's pointer. The provided builder can be used to create the default
|
|
/// value on the fly.
|
|
///
|
|
/// The builder is located at the beginning of the block where the slot
|
|
/// pointer is defined.
|
|
::mlir::Value getDefaultValue(const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
|
|
/// Hook triggered for every new block argument added to a block.
|
|
/// This will only be called for slots declared by this operation.
|
|
/// This function is called after removal of blocking uses, meaning
|
|
/// only operations that will be deleted remain users of the slot.
|
|
///
|
|
/// The builder is located at the beginning of the block on call. All IR
|
|
/// mutations must happen through the builder.
|
|
void handleBlockArgument(const ::mlir::MemorySlot & slot, ::mlir::BlockArgument argument, ::mlir::OpBuilder & builder);
|
|
|
|
/// Hook triggered once the promotion of a slot is complete. This can
|
|
/// also clean up the created default value if necessary. The default
|
|
/// value may be a null value if no default value was created.
|
|
/// This will only be called for slots declared by this operation.
|
|
///
|
|
/// Must return a new promotable allocation op if this operation produced
|
|
/// multiple promotable slots, nullopt otherwise.
|
|
::std::optional<::mlir::PromotableAllocationOpInterface> handlePromotionComplete(const ::mlir::MemorySlot & slot, ::mlir::Value defaultValue, ::mlir::OpBuilder & builder);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that can load from memory slots and/or store
|
|
/// to memory slots.
|
|
///
|
|
/// For a memory operation on a slot to be valid, it must strictly operate
|
|
/// within the bounds of the slot.
|
|
///
|
|
/// If the same operation does both loads and stores on the same slot, the
|
|
/// load must semantically happen first.
|
|
namespace detail {
|
|
struct PromotableMemOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
bool (*loadsFrom)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &);
|
|
bool (*storesTo)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &);
|
|
::mlir::Value (*getStored)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::OpBuilder &, ::mlir::Value, const ::mlir::DataLayout &);
|
|
bool (*canUsesBeRemoved)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &, ::llvm::SmallVectorImpl<::mlir::OpOperand *> &, const ::mlir::DataLayout &);
|
|
::mlir::DeletionKind (*removeBlockingUses)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &, ::mlir::OpBuilder &, ::mlir::Value, const ::mlir::DataLayout &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableMemOpInterface;
|
|
Model() : Concept{loadsFrom, storesTo, getStored, canUsesBeRemoved, removeBlockingUses} {}
|
|
|
|
static inline bool loadsFrom(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot);
|
|
static inline bool storesTo(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot);
|
|
static inline ::mlir::Value getStored(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder, ::mlir::Value reachingDef, const ::mlir::DataLayout & dataLayout);
|
|
static inline bool canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
static inline ::mlir::DeletionKind removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder, ::mlir::Value reachingDefinition, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableMemOpInterface;
|
|
FallbackModel() : Concept{loadsFrom, storesTo, getStored, canUsesBeRemoved, removeBlockingUses} {}
|
|
|
|
static inline bool loadsFrom(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot);
|
|
static inline bool storesTo(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot);
|
|
static inline ::mlir::Value getStored(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder, ::mlir::Value reachingDef, const ::mlir::DataLayout & dataLayout);
|
|
static inline bool canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
static inline ::mlir::DeletionKind removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder, ::mlir::Value reachingDefinition, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct PromotableMemOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class PromotableMemOpInterface : public ::mlir::OpInterface<PromotableMemOpInterface, detail::PromotableMemOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<PromotableMemOpInterface, detail::PromotableMemOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::PromotableMemOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Gets whether this operation loads from the specified slot.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
bool loadsFrom(const ::mlir::MemorySlot & slot);
|
|
|
|
/// Gets whether this operation stores to the specified slot.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
bool storesTo(const ::mlir::MemorySlot & slot);
|
|
|
|
/// Gets the value stored to the provided memory slot, or returns a null
|
|
/// value if this operation does not store to this slot. An operation
|
|
/// storing a value to a slot must always be able to provide the value it
|
|
/// stores. This method is only called once per slot promotion, and only
|
|
/// on operations that store to the slot according to the `storesTo` method.
|
|
///
|
|
/// The returned value must dominate the memory operation. This ensures
|
|
/// that new uses inserted by `visitReplacedValues` after the memory
|
|
/// operation are properly dominated.
|
|
///
|
|
/// The builder is positioned immediately before the memory operation on
|
|
/// call. No IR deletion is allowed in this method. IR mutations must not
|
|
/// introduce new uses of the memory slot. Existing control flow must not
|
|
/// be modified.
|
|
::mlir::Value getStored(const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder, ::mlir::Value reachingDef, const ::mlir::DataLayout & dataLayout);
|
|
|
|
/// Checks that this operation can be promoted to no longer use the provided
|
|
/// blocking uses, in the context of promoting `slot`.
|
|
///
|
|
/// If the removal procedure of the use will require that other uses get
|
|
/// removed, that dependency should be added to the `newBlockingUses`
|
|
/// argument. Dependent uses must only be uses of results of this operation.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
bool canUsesBeRemoved(const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
|
|
/// Transforms IR to ensure that the current operation does not use the
|
|
/// provided memory slot anymore. `reachingDefinition` contains the value
|
|
/// currently stored in the provided memory slot, immediately before the
|
|
/// current operation.
|
|
///
|
|
/// During the transformation, *no operation should be deleted*.
|
|
/// The operation can only schedule its own deletion by returning the
|
|
/// appropriate `DeletionKind`. The deletion must be legal assuming the
|
|
/// blocking uses passed through the `newBlockingUses` list in
|
|
/// `canUseBeRemoved` have been removed.
|
|
///
|
|
/// After calling this method, the blocking uses should have disappeared
|
|
/// or this operation should have scheduled its own deletion.
|
|
///
|
|
/// This method will only be called after ensuring promotion is allowed via
|
|
/// `canUseBeRemoved`. The requested blocking use removal may or may not
|
|
/// have been done at the point of calling this method, but it will be done
|
|
/// eventually.
|
|
///
|
|
/// The builder is located after the promotable operation on call.
|
|
::mlir::DeletionKind removeBlockingUses(const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder, ::mlir::Value reachingDefinition, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that can be transformed or deleted so it no longer
|
|
/// uses a provided value (blocking use), in case this would allow the promotion
|
|
/// of a memory slot.
|
|
namespace detail {
|
|
struct PromotableOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
bool (*canUsesBeRemoved)(const Concept *impl, ::mlir::Operation *, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &, ::llvm::SmallVectorImpl<::mlir::OpOperand *> &, const ::mlir::DataLayout &);
|
|
::mlir::DeletionKind (*removeBlockingUses)(const Concept *impl, ::mlir::Operation *, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &, ::mlir::OpBuilder &);
|
|
bool (*requiresReplacedValues)(const Concept *impl, ::mlir::Operation *);
|
|
void (*visitReplacedValues)(const Concept *impl, ::mlir::Operation *, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>>, ::mlir::OpBuilder &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableOpInterface;
|
|
Model() : Concept{canUsesBeRemoved, removeBlockingUses, requiresReplacedValues, visitReplacedValues} {}
|
|
|
|
static inline bool canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
static inline ::mlir::DeletionKind removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder);
|
|
static inline bool requiresReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline void visitReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableOpInterface;
|
|
FallbackModel() : Concept{canUsesBeRemoved, removeBlockingUses, requiresReplacedValues, visitReplacedValues} {}
|
|
|
|
static inline bool canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
static inline ::mlir::DeletionKind removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder);
|
|
static inline bool requiresReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline void visitReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
bool requiresReplacedValues(::mlir::Operation *tablegen_opaque_val) const;
|
|
void visitReplacedValues(::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder &builder) const;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct PromotableOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class PromotableOpInterface : public ::mlir::OpInterface<PromotableOpInterface, detail::PromotableOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<PromotableOpInterface, detail::PromotableOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::PromotableOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Checks that this operation can be promoted to no longer use the provided
|
|
/// blocking uses, in order to allow optimization.
|
|
///
|
|
/// If the removal procedure of the use will require that other uses get
|
|
/// removed, that dependency should be added to the `newBlockingUses`
|
|
/// argument. Dependent uses must only be uses of results of this operation.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
bool canUsesBeRemoved(const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout);
|
|
|
|
/// Transforms IR to ensure that the current operation does not use the
|
|
/// provided blocking uses anymore. In contrast to
|
|
/// `PromotableMemOpInterface`, operations implementing this interface
|
|
/// must not need access to the reaching definition of the content of the
|
|
/// slot.
|
|
///
|
|
/// During the transformation, *no operation should be deleted*.
|
|
/// The operation can only schedule its own deletion by returning the
|
|
/// appropriate `DeletionKind`. The deletion must be legal assuming the
|
|
/// blocking uses passed through the `newBlockingUses` list in
|
|
/// `canUseBeRemoved` have been removed.
|
|
///
|
|
/// After calling this method, the blocking uses should have disappeared
|
|
/// or this operation should have scheduled its own deletion.
|
|
///
|
|
/// This method will only be called after ensuring promotion is allowed via
|
|
/// `canUseBeRemoved`. The requested blocking use removal may or may not
|
|
/// have been done at the point of calling this method, but it will be done
|
|
/// eventually.
|
|
///
|
|
/// The builder is located after the promotable operation on call.
|
|
::mlir::DeletionKind removeBlockingUses(const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder);
|
|
|
|
/// Indicates whether the promoted operation needs to visit the SSA values
|
|
/// that replace the memory slot.
|
|
///
|
|
/// If true, `visitReplacedValues` will be called once for this operation
|
|
/// after all reaching definitions have been computed but before any
|
|
/// blocking uses are removed.
|
|
///
|
|
/// This should only return true if the intended transformation applies
|
|
/// to all replaced values. Replaced values must not be deleted.
|
|
bool requiresReplacedValues();
|
|
|
|
/// Transforms the IR using the SSA values replacing the memory slot.
|
|
///
|
|
/// Called after all reaching definitions for the slot have been computed
|
|
/// but before any blocking uses are removed, provided
|
|
/// `requiresReplacedValues` returns true. `mutatedDefs` contains the
|
|
/// replacing values. Any new uses of a load result created here will be
|
|
/// automatically redirected to its reaching definition when the load is
|
|
/// replaced.
|
|
///
|
|
/// The builder is positioned after the promotable operation.
|
|
/// No operations may be deleted during this transformation.
|
|
void visitReplacedValues(::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation that creates a transparent alias of a memory slot
|
|
/// accessed through one of its operands. Mem2Reg traverses chains of these
|
|
/// aliases to project slot values across them. This allows load and store
|
|
/// operations on the alias to be promoted as if they were directly accessing
|
|
/// the underlying slot.
|
|
///
|
|
/// Since an alias remains a blocking use of the underlying slot pointer, the
|
|
/// operation must also implement either `PromotableOpInterface` or
|
|
/// `PromotableMemOpInterface`. This ensures that mem2reg can remove the alias
|
|
/// after the slot has been promoted.
|
|
namespace detail {
|
|
struct PromotableAliaserInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
void (*getPromotableSlotAliases)(const Concept *impl, ::mlir::Operation *, ::mlir::OpOperand &, const ::mlir::MemorySlot &, ::llvm::SmallVectorImpl<::mlir::MemorySlot> &);
|
|
::mlir::Value (*projectSlotValueToAliasValue)(const Concept *impl, ::mlir::Operation *, ::mlir::OpOperand &, const ::mlir::MemorySlot &, const ::mlir::MemorySlot &, ::mlir::Value, ::mlir::OpBuilder &);
|
|
::mlir::Value (*projectAliasValueToSlotValue)(const Concept *impl, ::mlir::Operation *, ::mlir::OpOperand &, const ::mlir::MemorySlot &, const ::mlir::MemorySlot &, ::mlir::Value, ::mlir::Value, ::mlir::OpBuilder &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableAliaserInterface;
|
|
Model() : Concept{getPromotableSlotAliases, projectSlotValueToAliasValue, projectAliasValueToSlotValue} {}
|
|
|
|
static inline void getPromotableSlotAliases(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, ::llvm::SmallVectorImpl<::mlir::MemorySlot> & newMemorySlots);
|
|
static inline ::mlir::Value projectSlotValueToAliasValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder);
|
|
static inline ::mlir::Value projectAliasValueToSlotValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableAliaserInterface;
|
|
FallbackModel() : Concept{getPromotableSlotAliases, projectSlotValueToAliasValue, projectAliasValueToSlotValue} {}
|
|
|
|
static inline void getPromotableSlotAliases(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, ::llvm::SmallVectorImpl<::mlir::MemorySlot> & newMemorySlots);
|
|
static inline ::mlir::Value projectSlotValueToAliasValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder);
|
|
static inline ::mlir::Value projectAliasValueToSlotValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
::mlir::Value projectSlotValueToAliasValue(::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand &aliasedSlotPointerOperand, const ::mlir::MemorySlot &parentSlot, const ::mlir::MemorySlot &aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder &builder) const;
|
|
::mlir::Value projectAliasValueToSlotValue(::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand &aliasedSlotPointerOperand, const ::mlir::MemorySlot &parentSlot, const ::mlir::MemorySlot &aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder &builder) const;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct PromotableAliaserInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class PromotableAliaserInterface : public ::mlir::OpInterface<PromotableAliaserInterface, detail::PromotableAliaserInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<PromotableAliaserInterface, detail::PromotableAliaserInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::PromotableAliaserInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Populates `newMemorySlots` with the memory slots this operation
|
|
/// exposes by aliasing `parentSlot` (accessed via
|
|
/// `aliasedSlotPointerOperand`). Each new slot's pointer must be a
|
|
/// result of this operation, and its element type may differ from the
|
|
/// parent's. Leave the vector empty if no alias is exposed. An operation
|
|
/// can expose multiple aliases for the same parent slot.
|
|
///
|
|
/// `parentSlot` is provided so that aliasers using opaque pointers can
|
|
/// derive the new slot's element type from `parentSlot.elemType`.
|
|
///
|
|
/// Exposing an alias requires implementing the two projection methods
|
|
/// below to bridge values between the parent and new slot element types.
|
|
/// If these projections cannot be performed, `newMemorySlots` must be left
|
|
/// empty.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
void getPromotableSlotAliases(::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, ::llvm::SmallVectorImpl<::mlir::MemorySlot> & newMemorySlots);
|
|
|
|
/// Extracts the value of `aliasSlot` from `slotValue` (the value of
|
|
/// `parentSlot`). Mem2Reg invokes this method when a load on the new slot
|
|
/// requires the parent's value to be materialized with the new slot's
|
|
/// element type.
|
|
::mlir::Value projectSlotValueToAliasValue(::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder);
|
|
|
|
/// Reconstructs the value of `parentSlot` from `aliasValue` (a store to
|
|
/// `aliasSlot`) and `reachingDef` (the parent slot's value immediately
|
|
/// preceding the store). For full aliases, `reachingDef` can be ignored.
|
|
/// For partial sub-aliases, it allows the result to be constructed by
|
|
/// inserting `aliasValue` into `reachingDef`.
|
|
::mlir::Value projectAliasValueToSlotValue(::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes an operation for which memory slots can be promoted to SSA values
|
|
/// within the operation's regions.
|
|
namespace detail {
|
|
struct PromotableRegionOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
bool (*isRegionPromotable)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::Region *, bool);
|
|
void (*setupPromotion)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::Value, bool, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> &);
|
|
::mlir::Value (*finalizePromotion)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::Value, bool, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> &, ::mlir::OpBuilder &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableRegionOpInterface;
|
|
Model() : Concept{isRegionPromotable, setupPromotion, finalizePromotion} {}
|
|
|
|
static inline bool isRegionPromotable(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Region * region, bool hasValueStores);
|
|
static inline void setupPromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value reachingDef, bool hasValueStores, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> & regionsToProcess);
|
|
static inline ::mlir::Value finalizePromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value entryReachingDef, bool hasValueStores, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> & reachingAtBlockEnd, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::PromotableRegionOpInterface;
|
|
FallbackModel() : Concept{isRegionPromotable, setupPromotion, finalizePromotion} {}
|
|
|
|
static inline bool isRegionPromotable(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Region * region, bool hasValueStores);
|
|
static inline void setupPromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value reachingDef, bool hasValueStores, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> & regionsToProcess);
|
|
static inline ::mlir::Value finalizePromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value entryReachingDef, bool hasValueStores, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> & reachingAtBlockEnd, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct PromotableRegionOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class PromotableRegionOpInterface : public ::mlir::OpInterface<PromotableRegionOpInterface, detail::PromotableRegionOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<PromotableRegionOpInterface, detail::PromotableRegionOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::PromotableRegionOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Returns true when the provided region of the operation can be analyzed
|
|
/// for promotion. The provided region must be a child of the operation's
|
|
/// region.
|
|
/// The `hasValueStores` flag indicates whether the region contains
|
|
/// store-like operations that write to the memory slot.
|
|
bool isRegionPromotable(const ::mlir::MemorySlot & slot, ::mlir::Region * region, bool hasValueStores);
|
|
|
|
/// Called before processing the nested regions in the operation.
|
|
///
|
|
/// Based on the `reachingDef` value representing the value in the memory
|
|
/// slot at the entry into the operation, `setupPromotion` fills in the
|
|
/// `regionsToProcess` with the reaching definition at the entry of
|
|
/// all its promotable regions.
|
|
///
|
|
/// `setupPromotion` is allowed to mutate
|
|
/// the operation in place, including its nested regions, but cannot
|
|
/// delete existing operations or modify successor-bearing terminators.
|
|
/// Other mutations are not allowed.
|
|
///
|
|
/// The `hasValueStores` flag indicates whether the regions contain
|
|
/// `store`-like operations that write to the memory slot. This field can be
|
|
/// used to reduce the amount of book-keeping required to track the reaching
|
|
/// definitions.
|
|
void setupPromotion(const ::mlir::MemorySlot & slot, ::mlir::Value reachingDef, bool hasValueStores, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> & regionsToProcess);
|
|
|
|
/// Called once the reaching definitions have been computed for all the
|
|
/// regions, but before the actual removal of the blocking uses.
|
|
///
|
|
/// Returns the new reaching definition at the exit of the operation. For
|
|
/// this purpose, mutation of the operation is allowed under the following
|
|
/// constraints:
|
|
/// 1. If a region is deleted, all of its content must have been moved out
|
|
/// (not copied) to a new empty region that remains valid after the
|
|
/// deletion.
|
|
/// 2. Mutation must not change control flow within or between existing or
|
|
/// moved regions. This includes adding, removing or reordering blocks.
|
|
/// 3. Mutation must not modify or add operations that interact with the
|
|
/// value of the slot.
|
|
///
|
|
/// As an example, in order to add new results to the region operation, it
|
|
/// is allowed to clone the operation without regions, move (without
|
|
/// copying) the old region content into the new regions, and delete the
|
|
/// original operation.
|
|
///
|
|
/// The `entryReachingDef` is the reaching definition at the entry of the
|
|
/// region operation.
|
|
///
|
|
/// The `reachingAtBlockEnd` map contains the reaching definitions after all
|
|
/// the terminators within the regions of the operation. If a block of the
|
|
/// region is not present in the map, it is either dead code or within a
|
|
/// region that does not interact with the value of the slot.
|
|
///
|
|
/// The `hasValueStores` flag indicates whether the regions contain
|
|
/// `store`-like operations that write to the memory slot. This field can be
|
|
/// used to reduce the amount of book-keeping required to track the reaching
|
|
/// definitions.
|
|
::mlir::Value finalizePromotion(const ::mlir::MemorySlot & slot, ::mlir::Value entryReachingDef, bool hasValueStores, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> & reachingAtBlockEnd, ::mlir::OpBuilder & builder);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations allocating memory slots of aggregates that can be
|
|
/// destructured into multiple smaller allocations.
|
|
namespace detail {
|
|
struct DestructurableAllocationOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
::llvm::SmallVector<::mlir::DestructurableMemorySlot> (*getDestructurableSlots)(const Concept *impl, ::mlir::Operation *);
|
|
::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> (*destructure)(const Concept *impl, ::mlir::Operation *, const ::mlir::DestructurableMemorySlot &, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> &, ::mlir::OpBuilder &, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> &);
|
|
::std::optional<::mlir::DestructurableAllocationOpInterface> (*handleDestructuringComplete)(const Concept *impl, ::mlir::Operation *, const ::mlir::DestructurableMemorySlot &, ::mlir::OpBuilder &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::DestructurableAllocationOpInterface;
|
|
Model() : Concept{getDestructurableSlots, destructure, handleDestructuringComplete} {}
|
|
|
|
static inline ::llvm::SmallVector<::mlir::DestructurableMemorySlot> getDestructurableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> destructure(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::OpBuilder & builder, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> & newAllocators);
|
|
static inline ::std::optional<::mlir::DestructurableAllocationOpInterface> handleDestructuringComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::DestructurableAllocationOpInterface;
|
|
FallbackModel() : Concept{getDestructurableSlots, destructure, handleDestructuringComplete} {}
|
|
|
|
static inline ::llvm::SmallVector<::mlir::DestructurableMemorySlot> getDestructurableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val);
|
|
static inline ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> destructure(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::OpBuilder & builder, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> & newAllocators);
|
|
static inline ::std::optional<::mlir::DestructurableAllocationOpInterface> handleDestructuringComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct DestructurableAllocationOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class DestructurableAllocationOpInterface : public ::mlir::OpInterface<DestructurableAllocationOpInterface, detail::DestructurableAllocationOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<DestructurableAllocationOpInterface, detail::DestructurableAllocationOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::DestructurableAllocationOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Returns the list of slots for which destructuring should be attempted,
|
|
/// specifying in which way the slot should be destructured into subslots.
|
|
/// The subslots are indexed by attributes. This computes the type of the
|
|
/// pointer for each subslot to be generated. The type of the memory slot
|
|
/// must implement `DestructurableTypeInterface`.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
::llvm::SmallVector<::mlir::DestructurableMemorySlot> getDestructurableSlots();
|
|
|
|
/// Destructures this slot into multiple subslots. The newly generated slots
|
|
/// may belong to a different allocator. The original slot must still exist
|
|
/// at the end of this call. Only generates subslots for the indices found in
|
|
/// `usedIndices` since all other subslots are unused.
|
|
///
|
|
/// The builder is located at the beginning of the block where the slot
|
|
/// pointer is defined.
|
|
::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> destructure(const ::mlir::DestructurableMemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::OpBuilder & builder, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> & newAllocators);
|
|
|
|
/// Hook triggered once the destructuring of a slot is complete, meaning the
|
|
/// original slot is no longer being referred to and could be deleted.
|
|
/// This will only be called for slots declared by this operation.
|
|
///
|
|
/// Must return a new destructurable allocation op if this hook creates
|
|
/// a new destructurable op, nullopt otherwise.
|
|
::std::optional<::mlir::DestructurableAllocationOpInterface> handleDestructuringComplete(const ::mlir::DestructurableMemorySlot & slot, ::mlir::OpBuilder & builder);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations using memory slots in a safe manner.
|
|
namespace detail {
|
|
struct SafeMemorySlotAccessOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
::llvm::LogicalResult (*ensureOnlySafeAccesses)(const Concept *impl, ::mlir::Operation *, const ::mlir::MemorySlot &, ::mlir::SmallVectorImpl<::mlir::MemorySlot> &, const ::mlir::DataLayout &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::SafeMemorySlotAccessOpInterface;
|
|
Model() : Concept{ensureOnlySafeAccesses} {}
|
|
|
|
static inline ::llvm::LogicalResult ensureOnlySafeAccesses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::SafeMemorySlotAccessOpInterface;
|
|
FallbackModel() : Concept{ensureOnlySafeAccesses} {}
|
|
|
|
static inline ::llvm::LogicalResult ensureOnlySafeAccesses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct SafeMemorySlotAccessOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class SafeMemorySlotAccessOpInterface : public ::mlir::OpInterface<SafeMemorySlotAccessOpInterface, detail::SafeMemorySlotAccessOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<SafeMemorySlotAccessOpInterface, detail::SafeMemorySlotAccessOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::SafeMemorySlotAccessOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// Returns whether all accesses in this operation to the provided slot are
|
|
/// done in a safe manner. To be safe, the access must only access the slot
|
|
/// inside the bounds that its type implies.
|
|
///
|
|
/// If the safety of the accesses depends on the safety of the accesses to
|
|
/// further memory slots, the result of this method will be conditioned to
|
|
/// the safety of the accesses to the slots added by this method to
|
|
/// `mustBeSafelyUsed`.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
::llvm::LogicalResult ensureOnlySafeAccesses(const ::mlir::MemorySlot & slot, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
/// Describes operations that can access a sub-element of a destructurable slot.
|
|
namespace detail {
|
|
struct DestructurableAccessorOpInterfaceInterfaceTraits {
|
|
struct Concept {
|
|
/// The methods defined by the interface.
|
|
bool (*canRewire)(const Concept *impl, ::mlir::Operation *, const ::mlir::DestructurableMemorySlot &, ::llvm::SmallPtrSetImpl<::mlir::Attribute> &, ::mlir::SmallVectorImpl<::mlir::MemorySlot> &, const ::mlir::DataLayout &);
|
|
::mlir::DeletionKind (*rewire)(const Concept *impl, ::mlir::Operation *, const ::mlir::DestructurableMemorySlot &, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> &, ::mlir::OpBuilder &, const ::mlir::DataLayout &);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class Model : public Concept {
|
|
public:
|
|
using Interface = ::mlir::DestructurableAccessorOpInterface;
|
|
Model() : Concept{canRewire, rewire} {}
|
|
|
|
static inline bool canRewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
static inline ::mlir::DeletionKind rewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> & subslots, ::mlir::OpBuilder & builder, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteOp>
|
|
class FallbackModel : public Concept {
|
|
public:
|
|
using Interface = ::mlir::DestructurableAccessorOpInterface;
|
|
FallbackModel() : Concept{canRewire, rewire} {}
|
|
|
|
static inline bool canRewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
static inline ::mlir::DeletionKind rewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> & subslots, ::mlir::OpBuilder & builder, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
class ExternalModel : public FallbackModel<ConcreteModel> {
|
|
public:
|
|
using ConcreteEntity = ConcreteOp;
|
|
};
|
|
};
|
|
template <typename ConcreteOp>
|
|
struct DestructurableAccessorOpInterfaceTrait;
|
|
|
|
} // namespace detail
|
|
class DestructurableAccessorOpInterface : public ::mlir::OpInterface<DestructurableAccessorOpInterface, detail::DestructurableAccessorOpInterfaceInterfaceTraits> {
|
|
public:
|
|
using ::mlir::OpInterface<DestructurableAccessorOpInterface, detail::DestructurableAccessorOpInterfaceInterfaceTraits>::OpInterface;
|
|
template <typename ConcreteOp>
|
|
struct Trait : public detail::DestructurableAccessorOpInterfaceTrait<ConcreteOp> {};
|
|
|
|
/// For a given destructurable memory slot, returns whether this operation can
|
|
/// rewire its uses of the slot to use the slots generated after
|
|
/// destructuring. This may involve creating new operations.
|
|
///
|
|
/// This method must also register the indices it will access within the
|
|
/// `usedIndices` set. If the accessor generates new slots mapping to
|
|
/// subelements, they must be registered in `mustBeSafelyUsed` to ensure
|
|
/// they are used in a safe manner.
|
|
///
|
|
/// No IR mutation is allowed in this method.
|
|
bool canRewire(const ::mlir::DestructurableMemorySlot & slot, ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout);
|
|
|
|
/// Rewires the use of a slot to the generated subslots, without deleting
|
|
/// any operation. Returns whether the accessor should be deleted.
|
|
///
|
|
/// Deletion of operations is not allowed, only the accessor can be
|
|
/// scheduled for deletion by returning the appropriate value.
|
|
::mlir::DeletionKind rewire(const ::mlir::DestructurableMemorySlot & slot, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> & subslots, ::mlir::OpBuilder & builder, const ::mlir::DataLayout & dataLayout);
|
|
};
|
|
|
|
} // namespace mlir
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct PromotableAllocationOpInterfaceTrait : public ::mlir::OpInterface<PromotableAllocationOpInterface, detail::PromotableAllocationOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct PromotableMemOpInterfaceTrait : public ::mlir::OpInterface<PromotableMemOpInterface, detail::PromotableMemOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct PromotableOpInterfaceTrait : public ::mlir::OpInterface<PromotableOpInterface, detail::PromotableOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
|
|
/// Indicates whether the promoted operation needs to visit the SSA values
|
|
/// that replace the memory slot.
|
|
///
|
|
/// If true, `visitReplacedValues` will be called once for this operation
|
|
/// after all reaching definitions have been computed but before any
|
|
/// blocking uses are removed.
|
|
///
|
|
/// This should only return true if the intended transformation applies
|
|
/// to all replaced values. Replaced values must not be deleted.
|
|
bool requiresReplacedValues() {
|
|
return false;
|
|
}
|
|
|
|
/// Transforms the IR using the SSA values replacing the memory slot.
|
|
///
|
|
/// Called after all reaching definitions for the slot have been computed
|
|
/// but before any blocking uses are removed, provided
|
|
/// `requiresReplacedValues` returns true. `mutatedDefs` contains the
|
|
/// replacing values. Any new uses of a load result created here will be
|
|
/// automatically redirected to its reaching definition when the load is
|
|
/// replaced.
|
|
///
|
|
/// The builder is positioned after the promotable operation.
|
|
/// No operations may be deleted during this transformation.
|
|
void visitReplacedValues(::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder) {
|
|
return;
|
|
}
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct PromotableAliaserInterfaceTrait : public ::mlir::OpInterface<PromotableAliaserInterface, detail::PromotableAliaserInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
|
|
/// Extracts the value of `aliasSlot` from `slotValue` (the value of
|
|
/// `parentSlot`). Mem2Reg invokes this method when a load on the new slot
|
|
/// requires the parent's value to be materialized with the new slot's
|
|
/// element type.
|
|
::mlir::Value projectSlotValueToAliasValue(::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder) {
|
|
return slotValue;
|
|
}
|
|
|
|
/// Reconstructs the value of `parentSlot` from `aliasValue` (a store to
|
|
/// `aliasSlot`) and `reachingDef` (the parent slot's value immediately
|
|
/// preceding the store). For full aliases, `reachingDef` can be ignored.
|
|
/// For partial sub-aliases, it allows the result to be constructed by
|
|
/// inserting `aliasValue` into `reachingDef`.
|
|
::mlir::Value projectAliasValueToSlotValue(::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder) {
|
|
return aliasValue;
|
|
}
|
|
static ::llvm::LogicalResult verifyTrait(::mlir::Operation *op) {
|
|
if (!::mlir::isa<::mlir::PromotableOpInterface,
|
|
::mlir::PromotableMemOpInterface>(op))
|
|
return op->emitOpError(
|
|
"implements `PromotableAliaserInterface` but must also implement "
|
|
"`PromotableOpInterface` or `PromotableMemOpInterface`.");
|
|
return ::mlir::success();
|
|
}
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct PromotableRegionOpInterfaceTrait : public ::mlir::OpInterface<PromotableRegionOpInterface, detail::PromotableRegionOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct DestructurableAllocationOpInterfaceTrait : public ::mlir::OpInterface<DestructurableAllocationOpInterface, detail::DestructurableAllocationOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct SafeMemorySlotAccessOpInterfaceTrait : public ::mlir::OpInterface<SafeMemorySlotAccessOpInterface, detail::SafeMemorySlotAccessOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir::detail {
|
|
|
|
template <typename ConcreteOp>
|
|
struct DestructurableAccessorOpInterfaceTrait : public ::mlir::OpInterface<DestructurableAccessorOpInterface, detail::DestructurableAccessorOpInterfaceInterfaceTraits>::Trait<ConcreteOp> {
|
|
};
|
|
|
|
} // namespace mlir::detail
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
::llvm::SmallVector<::mlir::MemorySlot> detail::PromotableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::getPromotableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).getPromotableSlots();
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::getDefaultValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).getDefaultValue(slot, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::handleBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::BlockArgument argument, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).handleBlockArgument(slot, argument, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::std::optional<::mlir::PromotableAllocationOpInterface> detail::PromotableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::handlePromotionComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value defaultValue, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).handlePromotionComplete(slot, defaultValue, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::llvm::SmallVector<::mlir::MemorySlot> detail::PromotableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::getPromotableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return static_cast<const ConcreteOp *>(impl)->getPromotableSlots(tablegen_opaque_val);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::getDefaultValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->getDefaultValue(tablegen_opaque_val, slot, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::handleBlockArgument(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::BlockArgument argument, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->handleBlockArgument(tablegen_opaque_val, slot, argument, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::std::optional<::mlir::PromotableAllocationOpInterface> detail::PromotableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::handlePromotionComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value defaultValue, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->handlePromotionComplete(tablegen_opaque_val, slot, defaultValue, builder);
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::Model<ConcreteOp>::loadsFrom(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).loadsFrom(slot);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::Model<ConcreteOp>::storesTo(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).storesTo(slot);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableMemOpInterfaceInterfaceTraits::Model<ConcreteOp>::getStored(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder, ::mlir::Value reachingDef, const ::mlir::DataLayout & dataLayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).getStored(slot, builder, reachingDef, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::Model<ConcreteOp>::canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).canUsesBeRemoved(slot, blockingUses, newBlockingUses, datalayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::PromotableMemOpInterfaceInterfaceTraits::Model<ConcreteOp>::removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder, ::mlir::Value reachingDefinition, const ::mlir::DataLayout & dataLayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).removeBlockingUses(slot, blockingUses, builder, reachingDefinition, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::loadsFrom(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot) {
|
|
return static_cast<const ConcreteOp *>(impl)->loadsFrom(tablegen_opaque_val, slot);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::storesTo(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot) {
|
|
return static_cast<const ConcreteOp *>(impl)->storesTo(tablegen_opaque_val, slot);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableMemOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::getStored(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::OpBuilder & builder, ::mlir::Value reachingDef, const ::mlir::DataLayout & dataLayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->getStored(tablegen_opaque_val, slot, builder, reachingDef, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableMemOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->canUsesBeRemoved(tablegen_opaque_val, slot, blockingUses, newBlockingUses, datalayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::PromotableMemOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder, ::mlir::Value reachingDefinition, const ::mlir::DataLayout & dataLayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->removeBlockingUses(tablegen_opaque_val, slot, blockingUses, builder, reachingDefinition, dataLayout);
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableOpInterfaceInterfaceTraits::Model<ConcreteOp>::canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).canUsesBeRemoved(blockingUses, newBlockingUses, datalayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::PromotableOpInterfaceInterfaceTraits::Model<ConcreteOp>::removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).removeBlockingUses(blockingUses, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableOpInterfaceInterfaceTraits::Model<ConcreteOp>::requiresReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).requiresReplacedValues();
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableOpInterfaceInterfaceTraits::Model<ConcreteOp>::visitReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).visitReplacedValues(mutatedDefs, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::canUsesBeRemoved(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> & blockingUses, ::llvm::SmallVectorImpl<::mlir::OpOperand *> & newBlockingUses, const ::mlir::DataLayout & datalayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->canUsesBeRemoved(tablegen_opaque_val, blockingUses, newBlockingUses, datalayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::PromotableOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::removeBlockingUses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> & blockingUses, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->removeBlockingUses(tablegen_opaque_val, blockingUses, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::requiresReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return static_cast<const ConcreteOp *>(impl)->requiresReplacedValues(tablegen_opaque_val);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::visitReplacedValues(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->visitReplacedValues(tablegen_opaque_val, mutatedDefs, builder);
|
|
}
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
bool detail::PromotableOpInterfaceInterfaceTraits::ExternalModel<ConcreteModel, ConcreteOp>::requiresReplacedValues(::mlir::Operation *tablegen_opaque_val) const {
|
|
return false;
|
|
}
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
void detail::PromotableOpInterfaceInterfaceTraits::ExternalModel<ConcreteModel, ConcreteOp>::visitReplacedValues(::mlir::Operation *tablegen_opaque_val, ::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>> mutatedDefs, ::mlir::OpBuilder &builder) const {
|
|
return;
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableAliaserInterfaceInterfaceTraits::Model<ConcreteOp>::getPromotableSlotAliases(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, ::llvm::SmallVectorImpl<::mlir::MemorySlot> & newMemorySlots) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).getPromotableSlotAliases(aliasedSlotPointerOperand, parentSlot, newMemorySlots);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::Model<ConcreteOp>::projectSlotValueToAliasValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).projectSlotValueToAliasValue(aliasedSlotPointerOperand, parentSlot, aliasSlot, slotValue, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::Model<ConcreteOp>::projectAliasValueToSlotValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).projectAliasValueToSlotValue(aliasedSlotPointerOperand, parentSlot, aliasSlot, aliasValue, reachingDef, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableAliaserInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::getPromotableSlotAliases(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, ::llvm::SmallVectorImpl<::mlir::MemorySlot> & newMemorySlots) {
|
|
return static_cast<const ConcreteOp *>(impl)->getPromotableSlotAliases(tablegen_opaque_val, aliasedSlotPointerOperand, parentSlot, newMemorySlots);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::projectSlotValueToAliasValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->projectSlotValueToAliasValue(tablegen_opaque_val, aliasedSlotPointerOperand, parentSlot, aliasSlot, slotValue, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::projectAliasValueToSlotValue(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand & aliasedSlotPointerOperand, const ::mlir::MemorySlot & parentSlot, const ::mlir::MemorySlot & aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->projectAliasValueToSlotValue(tablegen_opaque_val, aliasedSlotPointerOperand, parentSlot, aliasSlot, aliasValue, reachingDef, builder);
|
|
}
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::ExternalModel<ConcreteModel, ConcreteOp>::projectSlotValueToAliasValue(::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand &aliasedSlotPointerOperand, const ::mlir::MemorySlot &parentSlot, const ::mlir::MemorySlot &aliasSlot, ::mlir::Value slotValue, ::mlir::OpBuilder &builder) const {
|
|
return slotValue;
|
|
}
|
|
template<typename ConcreteModel, typename ConcreteOp>
|
|
::mlir::Value detail::PromotableAliaserInterfaceInterfaceTraits::ExternalModel<ConcreteModel, ConcreteOp>::projectAliasValueToSlotValue(::mlir::Operation *tablegen_opaque_val, ::mlir::OpOperand &aliasedSlotPointerOperand, const ::mlir::MemorySlot &parentSlot, const ::mlir::MemorySlot &aliasSlot, ::mlir::Value aliasValue, ::mlir::Value reachingDef, ::mlir::OpBuilder &builder) const {
|
|
return aliasValue;
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableRegionOpInterfaceInterfaceTraits::Model<ConcreteOp>::isRegionPromotable(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Region * region, bool hasValueStores) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).isRegionPromotable(slot, region, hasValueStores);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableRegionOpInterfaceInterfaceTraits::Model<ConcreteOp>::setupPromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value reachingDef, bool hasValueStores, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> & regionsToProcess) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).setupPromotion(slot, reachingDef, hasValueStores, regionsToProcess);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableRegionOpInterfaceInterfaceTraits::Model<ConcreteOp>::finalizePromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value entryReachingDef, bool hasValueStores, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> & reachingAtBlockEnd, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).finalizePromotion(slot, entryReachingDef, hasValueStores, reachingAtBlockEnd, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::PromotableRegionOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::isRegionPromotable(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Region * region, bool hasValueStores) {
|
|
return static_cast<const ConcreteOp *>(impl)->isRegionPromotable(tablegen_opaque_val, slot, region, hasValueStores);
|
|
}
|
|
template<typename ConcreteOp>
|
|
void detail::PromotableRegionOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::setupPromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value reachingDef, bool hasValueStores, ::llvm::SmallMapVector<::mlir::Region *, ::mlir::Value, 2> & regionsToProcess) {
|
|
return static_cast<const ConcreteOp *>(impl)->setupPromotion(tablegen_opaque_val, slot, reachingDef, hasValueStores, regionsToProcess);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::Value detail::PromotableRegionOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::finalizePromotion(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::Value entryReachingDef, bool hasValueStores, const ::llvm::DenseMap<::mlir::Block *, ::mlir::Value> & reachingAtBlockEnd, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->finalizePromotion(tablegen_opaque_val, slot, entryReachingDef, hasValueStores, reachingAtBlockEnd, builder);
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
::llvm::SmallVector<::mlir::DestructurableMemorySlot> detail::DestructurableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::getDestructurableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).getDestructurableSlots();
|
|
}
|
|
template<typename ConcreteOp>
|
|
::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> detail::DestructurableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::destructure(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::OpBuilder & builder, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> & newAllocators) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).destructure(slot, usedIndices, builder, newAllocators);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::std::optional<::mlir::DestructurableAllocationOpInterface> detail::DestructurableAllocationOpInterfaceInterfaceTraits::Model<ConcreteOp>::handleDestructuringComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::mlir::OpBuilder & builder) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).handleDestructuringComplete(slot, builder);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::llvm::SmallVector<::mlir::DestructurableMemorySlot> detail::DestructurableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::getDestructurableSlots(const Concept *impl, ::mlir::Operation *tablegen_opaque_val) {
|
|
return static_cast<const ConcreteOp *>(impl)->getDestructurableSlots(tablegen_opaque_val);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> detail::DestructurableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::destructure(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, const ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::OpBuilder & builder, ::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> & newAllocators) {
|
|
return static_cast<const ConcreteOp *>(impl)->destructure(tablegen_opaque_val, slot, usedIndices, builder, newAllocators);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::std::optional<::mlir::DestructurableAllocationOpInterface> detail::DestructurableAllocationOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::handleDestructuringComplete(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::mlir::OpBuilder & builder) {
|
|
return static_cast<const ConcreteOp *>(impl)->handleDestructuringComplete(tablegen_opaque_val, slot, builder);
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
::llvm::LogicalResult detail::SafeMemorySlotAccessOpInterfaceInterfaceTraits::Model<ConcreteOp>::ensureOnlySafeAccesses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).ensureOnlySafeAccesses(slot, mustBeSafelyUsed, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::llvm::LogicalResult detail::SafeMemorySlotAccessOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::ensureOnlySafeAccesses(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::MemorySlot & slot, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->ensureOnlySafeAccesses(tablegen_opaque_val, slot, mustBeSafelyUsed, dataLayout);
|
|
}
|
|
|
|
} // namespace mlir
|
|
namespace mlir {
|
|
|
|
template<typename ConcreteOp>
|
|
bool detail::DestructurableAccessorOpInterfaceInterfaceTraits::Model<ConcreteOp>::canRewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).canRewire(slot, usedIndices, mustBeSafelyUsed, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::DestructurableAccessorOpInterfaceInterfaceTraits::Model<ConcreteOp>::rewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> & subslots, ::mlir::OpBuilder & builder, const ::mlir::DataLayout & dataLayout) {
|
|
return (llvm::cast<ConcreteOp>(tablegen_opaque_val)).rewire(slot, subslots, builder, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
bool detail::DestructurableAccessorOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::canRewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::SmallPtrSetImpl<::mlir::Attribute> & usedIndices, ::mlir::SmallVectorImpl<::mlir::MemorySlot> & mustBeSafelyUsed, const ::mlir::DataLayout & dataLayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->canRewire(tablegen_opaque_val, slot, usedIndices, mustBeSafelyUsed, dataLayout);
|
|
}
|
|
template<typename ConcreteOp>
|
|
::mlir::DeletionKind detail::DestructurableAccessorOpInterfaceInterfaceTraits::FallbackModel<ConcreteOp>::rewire(const Concept *impl, ::mlir::Operation *tablegen_opaque_val, const ::mlir::DestructurableMemorySlot & slot, ::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> & subslots, ::mlir::OpBuilder & builder, const ::mlir::DataLayout & dataLayout) {
|
|
return static_cast<const ConcreteOp *>(impl)->rewire(tablegen_opaque_val, slot, subslots, builder, dataLayout);
|
|
}
|
|
|
|
} // namespace mlir
|