1538 lines
60 KiB
C++
1538 lines
60 KiB
C++
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
|
|
|* *|
|
|
|* Enum Utility Declarations *|
|
|
|* *|
|
|
|* Automatically generated file, do not edit! *|
|
|
|* From: ROCDLOps.td *|
|
|
|* *|
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// ROCDL buffer out-of-bounds mode
|
|
enum class BufferOOBMode : uint32_t {
|
|
Any = 0,
|
|
Relaxed = 1,
|
|
Strict = 2,
|
|
};
|
|
|
|
::std::optional<BufferOOBMode> symbolizeBufferOOBMode(uint32_t);
|
|
::llvm::StringRef stringifyBufferOOBMode(BufferOOBMode);
|
|
::std::optional<BufferOOBMode> symbolizeBufferOOBMode(::llvm::StringRef);
|
|
inline constexpr unsigned getMaxEnumValForBufferOOBMode() {
|
|
return 2;
|
|
}
|
|
|
|
|
|
inline ::llvm::StringRef stringifyEnum(BufferOOBMode enumValue) {
|
|
return stringifyBufferOOBMode(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<BufferOOBMode> symbolizeEnum<BufferOOBMode>(::llvm::StringRef str) {
|
|
return symbolizeBufferOOBMode(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::BufferOOBMode, ::mlir::ROCDL::BufferOOBMode> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::BufferOOBMode> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
|
|
return parser.emitError(loc, "expected keyword for ROCDL buffer out-of-bounds mode");
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::BufferOOBMode> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::BufferOOBMode>(enumKeyword))
|
|
return *attr;
|
|
return parser.emitError(loc, "expected one of [any, relaxed, strict] for ROCDL buffer out-of-bounds mode, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::BufferOOBMode>, std::optional<::mlir::ROCDL::BufferOOBMode>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::BufferOOBMode>> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword, {"any", "relaxed", "strict"})))
|
|
return std::optional<::mlir::ROCDL::BufferOOBMode>{};
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::BufferOOBMode> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::BufferOOBMode>(enumKeyword))
|
|
return attr;
|
|
return parser.emitError(loc, "expected one of [any, relaxed, strict] for ROCDL buffer out-of-bounds mode, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::BufferOOBMode value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::BufferOOBMode> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::BufferOOBMode &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::BufferOOBMode &lhs, const ::mlir::ROCDL::BufferOOBMode &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// gfx12 atomic cache policy bits
|
|
enum class Gfx12AtomicCachePolicy : uint32_t {
|
|
none = 0,
|
|
keep_return = 1,
|
|
nt = 2,
|
|
cascade = 4,
|
|
scope_se = 8,
|
|
scope_dev = 16,
|
|
scope_sys = 24,
|
|
nv = 32,
|
|
volatile_op = 2147483648,
|
|
};
|
|
|
|
::std::optional<Gfx12AtomicCachePolicy> symbolizeGfx12AtomicCachePolicy(uint32_t);
|
|
std::string stringifyGfx12AtomicCachePolicy(Gfx12AtomicCachePolicy);
|
|
::std::optional<Gfx12AtomicCachePolicy> symbolizeGfx12AtomicCachePolicy(::llvm::StringRef);
|
|
|
|
inline constexpr Gfx12AtomicCachePolicy operator|(Gfx12AtomicCachePolicy a, Gfx12AtomicCachePolicy b) {
|
|
return static_cast<Gfx12AtomicCachePolicy>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy operator&(Gfx12AtomicCachePolicy a, Gfx12AtomicCachePolicy b) {
|
|
return static_cast<Gfx12AtomicCachePolicy>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy operator^(Gfx12AtomicCachePolicy a, Gfx12AtomicCachePolicy b) {
|
|
return static_cast<Gfx12AtomicCachePolicy>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy &operator|=(Gfx12AtomicCachePolicy &a, Gfx12AtomicCachePolicy b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy &operator&=(Gfx12AtomicCachePolicy &a, Gfx12AtomicCachePolicy b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy &operator^=(Gfx12AtomicCachePolicy &a, Gfx12AtomicCachePolicy b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy operator~(Gfx12AtomicCachePolicy bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<Gfx12AtomicCachePolicy>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(2147483711u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(Gfx12AtomicCachePolicy bits, Gfx12AtomicCachePolicy bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(Gfx12AtomicCachePolicy bits, Gfx12AtomicCachePolicy bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy bitEnumClear(Gfx12AtomicCachePolicy bits, Gfx12AtomicCachePolicy bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr Gfx12AtomicCachePolicy bitEnumSet(Gfx12AtomicCachePolicy bits, Gfx12AtomicCachePolicy bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(Gfx12AtomicCachePolicy enumValue) {
|
|
return stringifyGfx12AtomicCachePolicy(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<Gfx12AtomicCachePolicy> symbolizeEnum<Gfx12AtomicCachePolicy>(::llvm::StringRef str) {
|
|
return symbolizeGfx12AtomicCachePolicy(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::Gfx12AtomicCachePolicy, ::mlir::ROCDL::Gfx12AtomicCachePolicy> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::Gfx12AtomicCachePolicy> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx12AtomicCachePolicy flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for gfx12 atomic cache policy bits");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx12AtomicCachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, return, nt, cascade, scope_se, scope_dev, scope_sys, nv, volatile] for gfx12 atomic cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy>, std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx12AtomicCachePolicy flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "return", "nt", "cascade", "scope_se", "scope_dev", "scope_sys", "nv", "volatile"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy>{};
|
|
return parser.emitError(loc, "expected keyword for gfx12 atomic cache policy bits after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx12AtomicCachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, return, nt, cascade, scope_se, scope_dev, scope_sys, nv, volatile] for gfx12 atomic cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::Gfx12AtomicCachePolicy>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::Gfx12AtomicCachePolicy value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::Gfx12AtomicCachePolicy> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::Gfx12AtomicCachePolicy &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::Gfx12AtomicCachePolicy &lhs, const ::mlir::ROCDL::Gfx12AtomicCachePolicy &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// gfx12 cache policy bits
|
|
enum class Gfx12CachePolicy : uint32_t {
|
|
none = 0,
|
|
nt = 1,
|
|
ht = 2,
|
|
lu = 3,
|
|
nt_rt = 4,
|
|
rt_nt = 5,
|
|
nt_ht = 6,
|
|
nt_wb = 7,
|
|
scope_se = 8,
|
|
scope_dev = 16,
|
|
scope_sys = 24,
|
|
nv = 32,
|
|
swz = 64,
|
|
scal = 2048,
|
|
volatile_op = 2147483648,
|
|
};
|
|
|
|
::std::optional<Gfx12CachePolicy> symbolizeGfx12CachePolicy(uint32_t);
|
|
std::string stringifyGfx12CachePolicy(Gfx12CachePolicy);
|
|
::std::optional<Gfx12CachePolicy> symbolizeGfx12CachePolicy(::llvm::StringRef);
|
|
|
|
inline constexpr Gfx12CachePolicy operator|(Gfx12CachePolicy a, Gfx12CachePolicy b) {
|
|
return static_cast<Gfx12CachePolicy>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12CachePolicy operator&(Gfx12CachePolicy a, Gfx12CachePolicy b) {
|
|
return static_cast<Gfx12CachePolicy>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12CachePolicy operator^(Gfx12CachePolicy a, Gfx12CachePolicy b) {
|
|
return static_cast<Gfx12CachePolicy>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx12CachePolicy &operator|=(Gfx12CachePolicy &a, Gfx12CachePolicy b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr Gfx12CachePolicy &operator&=(Gfx12CachePolicy &a, Gfx12CachePolicy b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr Gfx12CachePolicy &operator^=(Gfx12CachePolicy &a, Gfx12CachePolicy b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr Gfx12CachePolicy operator~(Gfx12CachePolicy bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<Gfx12CachePolicy>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(2147485823u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(Gfx12CachePolicy bits, Gfx12CachePolicy bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(Gfx12CachePolicy bits, Gfx12CachePolicy bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr Gfx12CachePolicy bitEnumClear(Gfx12CachePolicy bits, Gfx12CachePolicy bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr Gfx12CachePolicy bitEnumSet(Gfx12CachePolicy bits, Gfx12CachePolicy bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(Gfx12CachePolicy enumValue) {
|
|
return stringifyGfx12CachePolicy(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<Gfx12CachePolicy> symbolizeEnum<Gfx12CachePolicy>(::llvm::StringRef str) {
|
|
return symbolizeGfx12CachePolicy(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::Gfx12CachePolicy, ::mlir::ROCDL::Gfx12CachePolicy> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::Gfx12CachePolicy> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx12CachePolicy flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for gfx12 cache policy bits");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx12CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx12CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, nt, ht, lu, nt_rt, rt_nt, nt_ht, nt_wb, scope_se, scope_dev, scope_sys, nv, swz, scal, volatile] for gfx12 cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::Gfx12CachePolicy>, std::optional<::mlir::ROCDL::Gfx12CachePolicy>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::Gfx12CachePolicy>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx12CachePolicy flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "nt", "ht", "lu", "nt_rt", "rt_nt", "nt_ht", "nt_wb", "scope_se", "scope_dev", "scope_sys", "nv", "swz", "scal", "volatile"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::Gfx12CachePolicy>{};
|
|
return parser.emitError(loc, "expected keyword for gfx12 cache policy bits after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx12CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx12CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, nt, ht, lu, nt_rt, rt_nt, nt_ht, nt_wb, scope_se, scope_dev, scope_sys, nv, swz, scal, volatile] for gfx12 cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::Gfx12CachePolicy>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::Gfx12CachePolicy value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::Gfx12CachePolicy> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::Gfx12CachePolicy &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::Gfx12CachePolicy &lhs, const ::mlir::ROCDL::Gfx12CachePolicy &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// gfx942 buffer cache policy bits
|
|
enum class Gfx942CachePolicy : uint32_t {
|
|
none = 0,
|
|
sc0 = 1,
|
|
nt = 2,
|
|
swz = 8,
|
|
sc1 = 16,
|
|
volatile_op = 2147483648,
|
|
};
|
|
|
|
::std::optional<Gfx942CachePolicy> symbolizeGfx942CachePolicy(uint32_t);
|
|
std::string stringifyGfx942CachePolicy(Gfx942CachePolicy);
|
|
::std::optional<Gfx942CachePolicy> symbolizeGfx942CachePolicy(::llvm::StringRef);
|
|
|
|
inline constexpr Gfx942CachePolicy operator|(Gfx942CachePolicy a, Gfx942CachePolicy b) {
|
|
return static_cast<Gfx942CachePolicy>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx942CachePolicy operator&(Gfx942CachePolicy a, Gfx942CachePolicy b) {
|
|
return static_cast<Gfx942CachePolicy>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx942CachePolicy operator^(Gfx942CachePolicy a, Gfx942CachePolicy b) {
|
|
return static_cast<Gfx942CachePolicy>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr Gfx942CachePolicy &operator|=(Gfx942CachePolicy &a, Gfx942CachePolicy b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr Gfx942CachePolicy &operator&=(Gfx942CachePolicy &a, Gfx942CachePolicy b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr Gfx942CachePolicy &operator^=(Gfx942CachePolicy &a, Gfx942CachePolicy b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr Gfx942CachePolicy operator~(Gfx942CachePolicy bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<Gfx942CachePolicy>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(2147483675u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(Gfx942CachePolicy bits, Gfx942CachePolicy bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(Gfx942CachePolicy bits, Gfx942CachePolicy bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr Gfx942CachePolicy bitEnumClear(Gfx942CachePolicy bits, Gfx942CachePolicy bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr Gfx942CachePolicy bitEnumSet(Gfx942CachePolicy bits, Gfx942CachePolicy bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(Gfx942CachePolicy enumValue) {
|
|
return stringifyGfx942CachePolicy(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<Gfx942CachePolicy> symbolizeEnum<Gfx942CachePolicy>(::llvm::StringRef str) {
|
|
return symbolizeGfx942CachePolicy(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::Gfx942CachePolicy, ::mlir::ROCDL::Gfx942CachePolicy> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::Gfx942CachePolicy> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx942CachePolicy flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for gfx942 buffer cache policy bits");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx942CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx942CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, sc0, nt, swz, sc1, volatile] for gfx942 buffer cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::Gfx942CachePolicy>, std::optional<::mlir::ROCDL::Gfx942CachePolicy>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::Gfx942CachePolicy>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::Gfx942CachePolicy flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "sc0", "nt", "swz", "sc1", "volatile"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::Gfx942CachePolicy>{};
|
|
return parser.emitError(loc, "expected keyword for gfx942 buffer cache policy bits after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::Gfx942CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::Gfx942CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, sc0, nt, swz, sc1, volatile] for gfx942 buffer cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::Gfx942CachePolicy>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::Gfx942CachePolicy value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::Gfx942CachePolicy> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::Gfx942CachePolicy &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::Gfx942CachePolicy &lhs, const ::mlir::ROCDL::Gfx942CachePolicy &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// negation modifier bitfield for gfx94x double-precision MFMA
|
|
enum class MFMANegModifier : uint32_t {
|
|
none = 0,
|
|
neg_a = 1,
|
|
neg_b = 2,
|
|
neg_c = 4,
|
|
};
|
|
|
|
::std::optional<MFMANegModifier> symbolizeMFMANegModifier(uint32_t);
|
|
std::string stringifyMFMANegModifier(MFMANegModifier);
|
|
::std::optional<MFMANegModifier> symbolizeMFMANegModifier(::llvm::StringRef);
|
|
|
|
inline constexpr MFMANegModifier operator|(MFMANegModifier a, MFMANegModifier b) {
|
|
return static_cast<MFMANegModifier>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr MFMANegModifier operator&(MFMANegModifier a, MFMANegModifier b) {
|
|
return static_cast<MFMANegModifier>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr MFMANegModifier operator^(MFMANegModifier a, MFMANegModifier b) {
|
|
return static_cast<MFMANegModifier>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr MFMANegModifier &operator|=(MFMANegModifier &a, MFMANegModifier b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr MFMANegModifier &operator&=(MFMANegModifier &a, MFMANegModifier b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr MFMANegModifier &operator^=(MFMANegModifier &a, MFMANegModifier b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr MFMANegModifier operator~(MFMANegModifier bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<MFMANegModifier>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(7u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(MFMANegModifier bits, MFMANegModifier bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(MFMANegModifier bits, MFMANegModifier bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr MFMANegModifier bitEnumClear(MFMANegModifier bits, MFMANegModifier bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr MFMANegModifier bitEnumSet(MFMANegModifier bits, MFMANegModifier bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(MFMANegModifier enumValue) {
|
|
return stringifyMFMANegModifier(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<MFMANegModifier> symbolizeEnum<MFMANegModifier>(::llvm::StringRef str) {
|
|
return symbolizeMFMANegModifier(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::MFMANegModifier, ::mlir::ROCDL::MFMANegModifier> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::MFMANegModifier> parse(ParserT &parser) {
|
|
::mlir::ROCDL::MFMANegModifier flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for negation modifier bitfield for gfx94x double-precision MFMA");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MFMANegModifier> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MFMANegModifier>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, neg_a, neg_b, neg_c] for negation modifier bitfield for gfx94x double-precision MFMA, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::MFMANegModifier>, std::optional<::mlir::ROCDL::MFMANegModifier>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::MFMANegModifier>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::MFMANegModifier flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "neg_a", "neg_b", "neg_c"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::MFMANegModifier>{};
|
|
return parser.emitError(loc, "expected keyword for negation modifier bitfield for gfx94x double-precision MFMA after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MFMANegModifier> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MFMANegModifier>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, neg_a, neg_b, neg_c] for negation modifier bitfield for gfx94x double-precision MFMA, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::MFMANegModifier>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::MFMANegModifier value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::MFMANegModifier> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::MFMANegModifier &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::MFMANegModifier &lhs, const ::mlir::ROCDL::MFMANegModifier &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// permutations of the lanes storing B in an MFMA
|
|
enum class MFMAPermB : uint32_t {
|
|
none = 0,
|
|
bcast_first_32 = 1,
|
|
bcast_second_32 = 2,
|
|
rotate_16_right = 3,
|
|
bcast_first_16 = 4,
|
|
bcast_second_16 = 5,
|
|
bcast_third_16 = 6,
|
|
bcast_fourth_16 = 7,
|
|
};
|
|
|
|
::std::optional<MFMAPermB> symbolizeMFMAPermB(uint32_t);
|
|
::llvm::StringRef stringifyMFMAPermB(MFMAPermB);
|
|
::std::optional<MFMAPermB> symbolizeMFMAPermB(::llvm::StringRef);
|
|
inline constexpr unsigned getMaxEnumValForMFMAPermB() {
|
|
return 7;
|
|
}
|
|
|
|
|
|
inline ::llvm::StringRef stringifyEnum(MFMAPermB enumValue) {
|
|
return stringifyMFMAPermB(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<MFMAPermB> symbolizeEnum<MFMAPermB>(::llvm::StringRef str) {
|
|
return symbolizeMFMAPermB(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::MFMAPermB, ::mlir::ROCDL::MFMAPermB> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::MFMAPermB> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
|
|
return parser.emitError(loc, "expected keyword for permutations of the lanes storing B in an MFMA");
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MFMAPermB> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MFMAPermB>(enumKeyword))
|
|
return *attr;
|
|
return parser.emitError(loc, "expected one of [none, bcast_first_32, bcast_second_32, rotate_16_right, bcast_first_16, bcast_second_16, bcast_third_16, bcast_fourth_16] for permutations of the lanes storing B in an MFMA, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::MFMAPermB>, std::optional<::mlir::ROCDL::MFMAPermB>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::MFMAPermB>> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword, {"none", "bcast_first_32", "bcast_second_32", "rotate_16_right", "bcast_first_16", "bcast_second_16", "bcast_third_16", "bcast_fourth_16"})))
|
|
return std::optional<::mlir::ROCDL::MFMAPermB>{};
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MFMAPermB> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MFMAPermB>(enumKeyword))
|
|
return attr;
|
|
return parser.emitError(loc, "expected one of [none, bcast_first_32, bcast_second_32, rotate_16_right, bcast_first_16, bcast_second_16, bcast_third_16, bcast_fourth_16] for permutations of the lanes storing B in an MFMA, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::MFMAPermB value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::MFMAPermB> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::MFMAPermB &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::MFMAPermB &lhs, const ::mlir::ROCDL::MFMAPermB &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// matrix operand formats selected by scaled MFMA/WMMA format fields
|
|
enum class MatrixFormat : uint32_t {
|
|
fp8_e4m3 = 0,
|
|
fp8_e5m2 = 1,
|
|
fp6_e2m3 = 2,
|
|
fp6_e3m2 = 3,
|
|
fp4_e2m1 = 4,
|
|
};
|
|
|
|
::std::optional<MatrixFormat> symbolizeMatrixFormat(uint32_t);
|
|
::llvm::StringRef stringifyMatrixFormat(MatrixFormat);
|
|
::std::optional<MatrixFormat> symbolizeMatrixFormat(::llvm::StringRef);
|
|
inline constexpr unsigned getMaxEnumValForMatrixFormat() {
|
|
return 4;
|
|
}
|
|
|
|
|
|
inline ::llvm::StringRef stringifyEnum(MatrixFormat enumValue) {
|
|
return stringifyMatrixFormat(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<MatrixFormat> symbolizeEnum<MatrixFormat>(::llvm::StringRef str) {
|
|
return symbolizeMatrixFormat(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::MatrixFormat, ::mlir::ROCDL::MatrixFormat> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::MatrixFormat> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
|
|
return parser.emitError(loc, "expected keyword for matrix operand formats selected by scaled MFMA/WMMA format fields");
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MatrixFormat> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MatrixFormat>(enumKeyword))
|
|
return *attr;
|
|
return parser.emitError(loc, "expected one of [fp8_e4m3, fp8_e5m2, fp6_e2m3, fp6_e3m2, fp4_e2m1] for matrix operand formats selected by scaled MFMA/WMMA format fields, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::MatrixFormat>, std::optional<::mlir::ROCDL::MatrixFormat>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::MatrixFormat>> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword, {"fp8_e4m3", "fp8_e5m2", "fp6_e2m3", "fp6_e3m2", "fp4_e2m1"})))
|
|
return std::optional<::mlir::ROCDL::MatrixFormat>{};
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::MatrixFormat> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::MatrixFormat>(enumKeyword))
|
|
return attr;
|
|
return parser.emitError(loc, "expected one of [fp8_e4m3, fp8_e5m2, fp6_e2m3, fp6_e3m2, fp4_e2m1] for matrix operand formats selected by scaled MFMA/WMMA format fields, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::MatrixFormat value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::MatrixFormat> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::MatrixFormat &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::MatrixFormat &lhs, const ::mlir::ROCDL::MatrixFormat &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// pre-gfx12 buffer cache policy bits
|
|
enum class PreGfx12CachePolicy : uint32_t {
|
|
none = 0,
|
|
glc = 1,
|
|
slc = 2,
|
|
dlc = 4,
|
|
swz = 8,
|
|
scc = 16,
|
|
all = 23,
|
|
volatile_op = 2147483648,
|
|
};
|
|
|
|
::std::optional<PreGfx12CachePolicy> symbolizePreGfx12CachePolicy(uint32_t);
|
|
std::string stringifyPreGfx12CachePolicy(PreGfx12CachePolicy);
|
|
::std::optional<PreGfx12CachePolicy> symbolizePreGfx12CachePolicy(::llvm::StringRef);
|
|
|
|
inline constexpr PreGfx12CachePolicy operator|(PreGfx12CachePolicy a, PreGfx12CachePolicy b) {
|
|
return static_cast<PreGfx12CachePolicy>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr PreGfx12CachePolicy operator&(PreGfx12CachePolicy a, PreGfx12CachePolicy b) {
|
|
return static_cast<PreGfx12CachePolicy>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr PreGfx12CachePolicy operator^(PreGfx12CachePolicy a, PreGfx12CachePolicy b) {
|
|
return static_cast<PreGfx12CachePolicy>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr PreGfx12CachePolicy &operator|=(PreGfx12CachePolicy &a, PreGfx12CachePolicy b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr PreGfx12CachePolicy &operator&=(PreGfx12CachePolicy &a, PreGfx12CachePolicy b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr PreGfx12CachePolicy &operator^=(PreGfx12CachePolicy &a, PreGfx12CachePolicy b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr PreGfx12CachePolicy operator~(PreGfx12CachePolicy bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<PreGfx12CachePolicy>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(2147483679u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(PreGfx12CachePolicy bits, PreGfx12CachePolicy bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(PreGfx12CachePolicy bits, PreGfx12CachePolicy bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr PreGfx12CachePolicy bitEnumClear(PreGfx12CachePolicy bits, PreGfx12CachePolicy bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr PreGfx12CachePolicy bitEnumSet(PreGfx12CachePolicy bits, PreGfx12CachePolicy bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(PreGfx12CachePolicy enumValue) {
|
|
return stringifyPreGfx12CachePolicy(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<PreGfx12CachePolicy> symbolizeEnum<PreGfx12CachePolicy>(::llvm::StringRef str) {
|
|
return symbolizePreGfx12CachePolicy(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::PreGfx12CachePolicy, ::mlir::ROCDL::PreGfx12CachePolicy> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::PreGfx12CachePolicy> parse(ParserT &parser) {
|
|
::mlir::ROCDL::PreGfx12CachePolicy flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for pre-gfx12 buffer cache policy bits");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::PreGfx12CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::PreGfx12CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, glc, slc, dlc, swz, scc, all, volatile] for pre-gfx12 buffer cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::PreGfx12CachePolicy>, std::optional<::mlir::ROCDL::PreGfx12CachePolicy>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::PreGfx12CachePolicy>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::PreGfx12CachePolicy flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "glc", "slc", "dlc", "swz", "scc", "all", "volatile"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::PreGfx12CachePolicy>{};
|
|
return parser.emitError(loc, "expected keyword for pre-gfx12 buffer cache policy bits after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::PreGfx12CachePolicy> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::PreGfx12CachePolicy>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, glc, slc, dlc, swz, scc, all, volatile] for pre-gfx12 buffer cache policy bits, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::PreGfx12CachePolicy>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::PreGfx12CachePolicy value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::PreGfx12CachePolicy> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::PreGfx12CachePolicy &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::PreGfx12CachePolicy &lhs, const ::mlir::ROCDL::PreGfx12CachePolicy &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// instruction type mask for scheduling barriers
|
|
enum class SchedGroupMask : uint32_t {
|
|
none = 0,
|
|
non_mem_non_sideeffect = 1,
|
|
valu = 2,
|
|
salu = 4,
|
|
mfma_wmma = 8,
|
|
all_vmem = 16,
|
|
vmem_read = 32,
|
|
vmem_write = 64,
|
|
all_ds = 128,
|
|
ds_read = 256,
|
|
ds_write = 512,
|
|
transcendental = 1024,
|
|
ldsdma = 2048,
|
|
all = 4095,
|
|
};
|
|
|
|
::std::optional<SchedGroupMask> symbolizeSchedGroupMask(uint32_t);
|
|
std::string stringifySchedGroupMask(SchedGroupMask);
|
|
::std::optional<SchedGroupMask> symbolizeSchedGroupMask(::llvm::StringRef);
|
|
|
|
inline constexpr SchedGroupMask operator|(SchedGroupMask a, SchedGroupMask b) {
|
|
return static_cast<SchedGroupMask>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr SchedGroupMask operator&(SchedGroupMask a, SchedGroupMask b) {
|
|
return static_cast<SchedGroupMask>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr SchedGroupMask operator^(SchedGroupMask a, SchedGroupMask b) {
|
|
return static_cast<SchedGroupMask>(static_cast<uint32_t>(a) ^ static_cast<uint32_t>(b));
|
|
}
|
|
inline constexpr SchedGroupMask &operator|=(SchedGroupMask &a, SchedGroupMask b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr SchedGroupMask &operator&=(SchedGroupMask &a, SchedGroupMask b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr SchedGroupMask &operator^=(SchedGroupMask &a, SchedGroupMask b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr SchedGroupMask operator~(SchedGroupMask bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<SchedGroupMask>(~static_cast<uint32_t>(bits) & static_cast<uint32_t>(4095u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(SchedGroupMask bits, SchedGroupMask bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(SchedGroupMask bits, SchedGroupMask bit) {
|
|
return (static_cast<uint32_t>(bits) & static_cast<uint32_t>(bit)) != 0;
|
|
}
|
|
inline constexpr SchedGroupMask bitEnumClear(SchedGroupMask bits, SchedGroupMask bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr SchedGroupMask bitEnumSet(SchedGroupMask bits, SchedGroupMask bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(SchedGroupMask enumValue) {
|
|
return stringifySchedGroupMask(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<SchedGroupMask> symbolizeEnum<SchedGroupMask>(::llvm::StringRef str) {
|
|
return symbolizeSchedGroupMask(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::SchedGroupMask, ::mlir::ROCDL::SchedGroupMask> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::SchedGroupMask> parse(ParserT &parser) {
|
|
::mlir::ROCDL::SchedGroupMask flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for instruction type mask for scheduling barriers");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::SchedGroupMask> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::SchedGroupMask>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, non_mem_non_sideeffect, valu, salu, mfma_wmma, all_vmem, vmem_read, vmem_write, all_ds, ds_read, ds_write, transcendental, ldsdma, all] for instruction type mask for scheduling barriers, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::SchedGroupMask>, std::optional<::mlir::ROCDL::SchedGroupMask>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::SchedGroupMask>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::SchedGroupMask flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "non_mem_non_sideeffect", "valu", "salu", "mfma_wmma", "all_vmem", "vmem_read", "vmem_write", "all_ds", "ds_read", "ds_write", "transcendental", "ldsdma", "all"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::SchedGroupMask>{};
|
|
return parser.emitError(loc, "expected keyword for instruction type mask for scheduling barriers after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::SchedGroupMask> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::SchedGroupMask>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, non_mem_non_sideeffect, valu, salu, mfma_wmma, all_vmem, vmem_read, vmem_write, all_ds, ds_read, ds_write, transcendental, ldsdma, all] for instruction type mask for scheduling barriers, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::SchedGroupMask>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::SchedGroupMask value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::SchedGroupMask> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::SchedGroupMask &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::SchedGroupMask &lhs, const ::mlir::ROCDL::SchedGroupMask &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// WMMA C operand modifiers
|
|
enum class WMMACModifier : uint16_t {
|
|
none = 0,
|
|
neg = 1,
|
|
abs = 2,
|
|
neg_abs = 3,
|
|
};
|
|
|
|
::std::optional<WMMACModifier> symbolizeWMMACModifier(uint16_t);
|
|
std::string stringifyWMMACModifier(WMMACModifier);
|
|
::std::optional<WMMACModifier> symbolizeWMMACModifier(::llvm::StringRef);
|
|
|
|
inline constexpr WMMACModifier operator|(WMMACModifier a, WMMACModifier b) {
|
|
return static_cast<WMMACModifier>(static_cast<uint16_t>(a) | static_cast<uint16_t>(b));
|
|
}
|
|
inline constexpr WMMACModifier operator&(WMMACModifier a, WMMACModifier b) {
|
|
return static_cast<WMMACModifier>(static_cast<uint16_t>(a) & static_cast<uint16_t>(b));
|
|
}
|
|
inline constexpr WMMACModifier operator^(WMMACModifier a, WMMACModifier b) {
|
|
return static_cast<WMMACModifier>(static_cast<uint16_t>(a) ^ static_cast<uint16_t>(b));
|
|
}
|
|
inline constexpr WMMACModifier &operator|=(WMMACModifier &a, WMMACModifier b) {
|
|
return a = a | b;
|
|
}
|
|
inline constexpr WMMACModifier &operator&=(WMMACModifier &a, WMMACModifier b) {
|
|
return a = a & b;
|
|
}
|
|
inline constexpr WMMACModifier &operator^=(WMMACModifier &a, WMMACModifier b) {
|
|
return a = a ^ b;
|
|
}
|
|
inline constexpr WMMACModifier operator~(WMMACModifier bits) {
|
|
// Ensure only bits that can be present in the enum are set
|
|
return static_cast<WMMACModifier>(~static_cast<uint16_t>(bits) & static_cast<uint16_t>(3u));
|
|
}
|
|
inline constexpr bool bitEnumContainsAll(WMMACModifier bits, WMMACModifier bit) {
|
|
return (bits & bit) == bit;
|
|
}
|
|
inline constexpr bool bitEnumContainsAny(WMMACModifier bits, WMMACModifier bit) {
|
|
return (static_cast<uint16_t>(bits) & static_cast<uint16_t>(bit)) != 0;
|
|
}
|
|
inline constexpr WMMACModifier bitEnumClear(WMMACModifier bits, WMMACModifier bit) {
|
|
return bits & ~bit;
|
|
}
|
|
inline constexpr WMMACModifier bitEnumSet(WMMACModifier bits, WMMACModifier bit, /*optional*/bool value=true) {
|
|
return value ? (bits | bit) : bitEnumClear(bits, bit);
|
|
}
|
|
|
|
inline std::string stringifyEnum(WMMACModifier enumValue) {
|
|
return stringifyWMMACModifier(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<WMMACModifier> symbolizeEnum<WMMACModifier>(::llvm::StringRef str) {
|
|
return symbolizeWMMACModifier(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::WMMACModifier, ::mlir::ROCDL::WMMACModifier> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::WMMACModifier> parse(ParserT &parser) {
|
|
::mlir::ROCDL::WMMACModifier flags = {};
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword))) {
|
|
return parser.emitError(loc, "expected keyword for WMMA C operand modifiers");
|
|
}
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMACModifier> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMACModifier>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, neg, abs, neg_abs] for WMMA C operand modifiers, got: ") << enumKeyword;
|
|
}
|
|
} while (::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return flags;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::WMMACModifier>, std::optional<::mlir::ROCDL::WMMACModifier>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::WMMACModifier>> parse(ParserT &parser) {
|
|
::mlir::ROCDL::WMMACModifier flags = {};
|
|
bool firstIter = true;
|
|
do {
|
|
// Parse the keyword containing a part of the enum.
|
|
::llvm::StringRef enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeyword(&enumKeyword, {"none", "neg", "abs", "neg_abs"}))) {
|
|
if (firstIter)
|
|
return std::optional<::mlir::ROCDL::WMMACModifier>{};
|
|
return parser.emitError(loc, "expected keyword for WMMA C operand modifiers after '|'");
|
|
}
|
|
firstIter = false;
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMACModifier> flag = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMACModifier>(enumKeyword)) {
|
|
flags = flags | *flag;
|
|
} else {
|
|
return parser.emitError(loc, "expected one of [none, neg, abs, neg_abs] for WMMA C operand modifiers, got: ") << enumKeyword;
|
|
}
|
|
} while(::mlir::succeeded(parser.parseOptionalVerticalBar()));
|
|
return std::optional<::mlir::ROCDL::WMMACModifier>{flags};
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::WMMACModifier value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::WMMACModifier> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint16_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::WMMACModifier &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint16_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::WMMACModifier &lhs, const ::mlir::ROCDL::WMMACModifier &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// matrix scale row selector
|
|
enum class WMMAMatrixScale : uint32_t {
|
|
row0 = 0,
|
|
row1 = 1,
|
|
};
|
|
|
|
::std::optional<WMMAMatrixScale> symbolizeWMMAMatrixScale(uint32_t);
|
|
::llvm::StringRef stringifyWMMAMatrixScale(WMMAMatrixScale);
|
|
::std::optional<WMMAMatrixScale> symbolizeWMMAMatrixScale(::llvm::StringRef);
|
|
inline constexpr unsigned getMaxEnumValForWMMAMatrixScale() {
|
|
return 1;
|
|
}
|
|
|
|
|
|
inline ::llvm::StringRef stringifyEnum(WMMAMatrixScale enumValue) {
|
|
return stringifyWMMAMatrixScale(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<WMMAMatrixScale> symbolizeEnum<WMMAMatrixScale>(::llvm::StringRef str) {
|
|
return symbolizeWMMAMatrixScale(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::WMMAMatrixScale, ::mlir::ROCDL::WMMAMatrixScale> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::WMMAMatrixScale> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
|
|
return parser.emitError(loc, "expected keyword for matrix scale row selector");
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMAMatrixScale> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMAMatrixScale>(enumKeyword))
|
|
return *attr;
|
|
return parser.emitError(loc, "expected one of [row0, row1] for matrix scale row selector, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::WMMAMatrixScale>, std::optional<::mlir::ROCDL::WMMAMatrixScale>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::WMMAMatrixScale>> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword, {"row0", "row1"})))
|
|
return std::optional<::mlir::ROCDL::WMMAMatrixScale>{};
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMAMatrixScale> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMAMatrixScale>(enumKeyword))
|
|
return attr;
|
|
return parser.emitError(loc, "expected one of [row0, row1] for matrix scale row selector, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::WMMAMatrixScale value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::WMMAMatrixScale> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::WMMAMatrixScale &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::WMMAMatrixScale &lhs, const ::mlir::ROCDL::WMMAMatrixScale &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|
|
namespace mlir::ROCDL {
|
|
|
|
// matrix scale exponent formats
|
|
enum class WMMAMatrixScaleFormat : uint32_t {
|
|
e8 = 0,
|
|
e5m3 = 1,
|
|
e4m3 = 2,
|
|
};
|
|
|
|
::std::optional<WMMAMatrixScaleFormat> symbolizeWMMAMatrixScaleFormat(uint32_t);
|
|
::llvm::StringRef stringifyWMMAMatrixScaleFormat(WMMAMatrixScaleFormat);
|
|
::std::optional<WMMAMatrixScaleFormat> symbolizeWMMAMatrixScaleFormat(::llvm::StringRef);
|
|
inline constexpr unsigned getMaxEnumValForWMMAMatrixScaleFormat() {
|
|
return 2;
|
|
}
|
|
|
|
|
|
inline ::llvm::StringRef stringifyEnum(WMMAMatrixScaleFormat enumValue) {
|
|
return stringifyWMMAMatrixScaleFormat(enumValue);
|
|
}
|
|
|
|
template <typename EnumType>
|
|
::std::optional<EnumType> symbolizeEnum(::llvm::StringRef);
|
|
|
|
template <>
|
|
inline ::std::optional<WMMAMatrixScaleFormat> symbolizeEnum<WMMAMatrixScaleFormat>(::llvm::StringRef str) {
|
|
return symbolizeWMMAMatrixScaleFormat(str);
|
|
}
|
|
|
|
} // namespace mlir::ROCDL
|
|
|
|
namespace mlir {
|
|
template <typename T, typename>
|
|
struct FieldParser;
|
|
|
|
template<>
|
|
struct FieldParser<::mlir::ROCDL::WMMAMatrixScaleFormat, ::mlir::ROCDL::WMMAMatrixScaleFormat> {
|
|
template <typename ParserT>
|
|
static FailureOr<::mlir::ROCDL::WMMAMatrixScaleFormat> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
|
|
return parser.emitError(loc, "expected keyword for matrix scale exponent formats");
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMAMatrixScaleFormat>(enumKeyword))
|
|
return *attr;
|
|
return parser.emitError(loc, "expected one of [e8, e5m3, e4m3] for matrix scale exponent formats, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
|
|
/// Support for std::optional, useful in attribute/type definition where the enum is
|
|
/// used as:
|
|
///
|
|
/// let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
|
|
template<>
|
|
struct FieldParser<std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat>, std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat>> {
|
|
template <typename ParserT>
|
|
static FailureOr<std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat>> parse(ParserT &parser) {
|
|
// Parse the keyword/string containing the enum.
|
|
std::string enumKeyword;
|
|
auto loc = parser.getCurrentLocation();
|
|
if (failed(parser.parseOptionalKeywordOrString(&enumKeyword, {"e8", "e5m3", "e4m3"})))
|
|
return std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat>{};
|
|
|
|
// Symbolize the keyword.
|
|
if (::std::optional<::mlir::ROCDL::WMMAMatrixScaleFormat> attr = ::mlir::ROCDL::symbolizeEnum<::mlir::ROCDL::WMMAMatrixScaleFormat>(enumKeyword))
|
|
return attr;
|
|
return parser.emitError(loc, "expected one of [e8, e5m3, e4m3] for matrix scale exponent formats, got: ") << enumKeyword;
|
|
}
|
|
};
|
|
} // namespace mlir
|
|
|
|
namespace llvm {
|
|
inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, ::mlir::ROCDL::WMMAMatrixScaleFormat value) {
|
|
auto valueStr = stringifyEnum(value);
|
|
return p << valueStr;
|
|
}
|
|
} // namespace llvm
|
|
|
|
namespace llvm {
|
|
template<> struct DenseMapInfo<::mlir::ROCDL::WMMAMatrixScaleFormat> {
|
|
using StorageInfo = ::llvm::DenseMapInfo<uint32_t>;
|
|
|
|
static unsigned getHashValue(const ::mlir::ROCDL::WMMAMatrixScaleFormat &val) {
|
|
return StorageInfo::getHashValue(static_cast<uint32_t>(val));
|
|
}
|
|
|
|
static bool isEqual(const ::mlir::ROCDL::WMMAMatrixScaleFormat &lhs, const ::mlir::ROCDL::WMMAMatrixScaleFormat &rhs) {
|
|
return lhs == rhs;
|
|
}
|
|
};
|
|
}
|
|
|