278 lines
12 KiB
C++
278 lines
12 KiB
C++
//===- llvm/IR/ProfDataUtils.h - Profiling Metadata Utilities ---*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// @file
|
|
/// This file contains the declarations for profiling metadata utility
|
|
/// functions.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_IR_PROFDATAUTILS_H
|
|
#define LLVM_IR_PROFDATAUTILS_H
|
|
|
|
#include "llvm/ADT/STLFunctionalExtras.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/IR/Metadata.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Support/Compiler.h"
|
|
#include <cstddef>
|
|
#include <type_traits>
|
|
|
|
namespace llvm {
|
|
struct MDProfLabels {
|
|
LLVM_ABI static const char *BranchWeights;
|
|
LLVM_ABI static const char *ValueProfile;
|
|
LLVM_ABI static const char *FunctionEntryCount;
|
|
LLVM_ABI static const char *SyntheticFunctionEntryCount;
|
|
LLVM_ABI static const char *ExpectedBranchWeights;
|
|
LLVM_ABI static const char *UnknownBranchWeightsMarker;
|
|
};
|
|
|
|
extern LLVM_ABI cl::opt<bool> ProfcheckDisableMetadataFixes;
|
|
|
|
/// Profile-based loop metadata that should be accessed only by using
|
|
/// \c llvm::getLoopEstimatedTripCount and \c llvm::setLoopEstimatedTripCount.
|
|
LLVM_ABI extern const char *LLVMLoopEstimatedTripCount;
|
|
|
|
/// Checks if an Instruction has MD_prof Metadata
|
|
LLVM_ABI bool hasProfMD(const Instruction &I);
|
|
|
|
/// Checks if an MDNode contains Branch Weight Metadata
|
|
LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData);
|
|
|
|
/// Checks if an MDNode contains value profiling Metadata
|
|
LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData);
|
|
|
|
/// Checks if an instructions has Branch Weight Metadata
|
|
///
|
|
/// \param I The instruction to check
|
|
/// \returns True if I has an MD_prof node containing Branch Weights. False
|
|
/// otherwise.
|
|
LLVM_ABI bool hasBranchWeightMD(const Instruction &I);
|
|
|
|
/// Checks if an instructions has valid Branch Weight Metadata
|
|
///
|
|
/// \param I The instruction to check
|
|
/// \returns True if I has an MD_prof node containing valid Branch Weights,
|
|
/// i.e., one weight for each successor. False otherwise.
|
|
LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I);
|
|
|
|
/// Get the branch weights metadata node
|
|
///
|
|
/// \param I The Instruction to get the weights from.
|
|
/// \returns A pointer to I's branch weights metadata node, if it exists.
|
|
/// Nullptr otherwise.
|
|
LLVM_ABI MDNode *getBranchWeightMDNode(const Instruction &I);
|
|
|
|
/// Get the valid branch weights metadata node
|
|
///
|
|
/// \param I The Instruction to get the weights from.
|
|
/// \returns A pointer to I's valid branch weights metadata node, if it exists.
|
|
/// Nullptr otherwise.
|
|
LLVM_ABI MDNode *getValidBranchWeightMDNode(const Instruction &I);
|
|
|
|
/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
|
|
/// intrinsic
|
|
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I);
|
|
|
|
/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
|
|
/// intrinsic
|
|
LLVM_ABI bool hasBranchWeightOrigin(const MDNode *ProfileData);
|
|
|
|
/// Return the offset to the first branch weight data
|
|
LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData);
|
|
|
|
LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData);
|
|
|
|
/// Extract branch weights from MD_prof metadata
|
|
///
|
|
/// \param ProfileData A pointer to an MDNode.
|
|
/// \param [out] Weights An output vector to fill with branch weights
|
|
/// \returns True if weights were extracted, False otherwise. When false Weights
|
|
/// will be cleared.
|
|
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData,
|
|
SmallVectorImpl<uint32_t> &Weights);
|
|
|
|
/// Faster version of extractBranchWeights() that skips checks and must only
|
|
/// be called with "branch_weights" metadata nodes. Supports uint32_t.
|
|
LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData,
|
|
SmallVectorImpl<uint32_t> &Weights);
|
|
|
|
/// Faster version of extractBranchWeights() that skips checks and must only
|
|
/// be called with "branch_weights" metadata nodes. Supports uint64_t.
|
|
LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData,
|
|
SmallVectorImpl<uint64_t> &Weights);
|
|
|
|
/// Extract branch weights attatched to an Instruction
|
|
///
|
|
/// \param I The Instruction to extract weights from.
|
|
/// \param [out] Weights An output vector to fill with branch weights
|
|
/// \returns True if weights were extracted, False otherwise. When false Weights
|
|
/// will be cleared.
|
|
LLVM_ABI bool extractBranchWeights(const Instruction &I,
|
|
SmallVectorImpl<uint32_t> &Weights);
|
|
|
|
/// Extract branch weights from a conditional branch or select Instruction.
|
|
///
|
|
/// \param I The instruction to extract branch weights from.
|
|
/// \param [out] TrueVal will contain the branch weight for the True branch
|
|
/// \param [out] FalseVal will contain the branch weight for the False branch
|
|
/// \returns True on success with profile weights filled in. False if no
|
|
/// metadata or invalid metadata was found.
|
|
LLVM_ABI bool extractBranchWeights(const Instruction &I, uint64_t &TrueVal,
|
|
uint64_t &FalseVal);
|
|
|
|
/// Retrieve the total of all weights from MD_prof data.
|
|
///
|
|
/// \param ProfileData The profile data to extract the total weight from
|
|
/// \param [out] TotalWeights input variable to fill with total weights
|
|
/// \returns True on success with profile total weights filled in. False if no
|
|
/// metadata was found.
|
|
LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData,
|
|
uint64_t &TotalWeights);
|
|
|
|
/// Retrieve the total of all weights from an instruction.
|
|
///
|
|
/// \param I The instruction to extract the total weight from
|
|
/// \param [out] TotalWeights input variable to fill with total weights
|
|
/// \returns True on success with profile total weights filled in. False if no
|
|
/// metadata was found.
|
|
LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
|
|
uint64_t &TotalWeights);
|
|
|
|
/// Create a new `branch_weights` metadata node and add or overwrite
|
|
/// a `prof` metadata reference to instruction `I`.
|
|
/// \param I the Instruction to set branch weights on.
|
|
/// \param Weights an array of weights to set on instruction I.
|
|
/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
|
|
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
|
|
bool IsExpected, bool ElideAllZero = false);
|
|
|
|
/// Push the weights right to fit in uint32_t.
|
|
LLVM_ABI SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights);
|
|
|
|
/// Variant of `setBranchWeights` where the `Weights` will be fit first to
|
|
/// uint32_t by shifting right.
|
|
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef<uint64_t> Weights,
|
|
bool IsExpected,
|
|
bool ElideAllZero = false);
|
|
|
|
/// downscale the given weights preserving the ratio. If the maximum value is
|
|
/// not already known and not provided via \param KnownMaxCount , it will be
|
|
/// obtained from \param Weights.
|
|
LLVM_ABI SmallVector<uint32_t>
|
|
downscaleWeights(ArrayRef<uint64_t> Weights,
|
|
std::optional<uint64_t> KnownMaxCount = std::nullopt);
|
|
|
|
/// Calculate what to divide by to scale counts.
|
|
///
|
|
/// Given the maximum count, calculate a divisor that will scale all the
|
|
/// weights to strictly less than std::numeric_limits<uint32_t>::max().
|
|
inline uint64_t calculateCountScale(uint64_t MaxCount) {
|
|
return MaxCount < std::numeric_limits<uint32_t>::max()
|
|
? 1
|
|
: MaxCount / std::numeric_limits<uint32_t>::max() + 1;
|
|
}
|
|
|
|
/// Scale an individual branch count.
|
|
///
|
|
/// Scale a 64-bit weight down to 32-bits using \c Scale.
|
|
///
|
|
inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) {
|
|
uint64_t Scaled = Count / Scale;
|
|
assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
|
|
return Scaled;
|
|
}
|
|
|
|
/// Specify that the branch weights for this terminator cannot be known at
|
|
/// compile time. This should only be called by passes, and never as a default
|
|
/// behavior in e.g. MDBuilder. The goal is to use this info to validate passes
|
|
/// do not accidentally drop profile info, and this API is called in cases where
|
|
/// the pass explicitly cannot provide that info. Defaulting it in would hide
|
|
/// bugs where the pass forgets to transfer over or otherwise specify profile
|
|
/// info. Use `PassName` to capture the pass name (i.e. DEBUG_TYPE) for
|
|
/// debuggability.
|
|
LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I,
|
|
StringRef PassName);
|
|
|
|
/// Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch
|
|
/// weights in the new instruction if the parent function of the original
|
|
/// instruction has an entry count. This is to not confuse users by injecting
|
|
/// profile data into non-profiled functions. If \p F is nullptr, we will fetch
|
|
/// the function from \p I.
|
|
LLVM_ABI void
|
|
setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName,
|
|
const Function *F = nullptr);
|
|
|
|
/// Returns a metadata node containing unknown branch weights if the function
|
|
/// has an entry count, otherwise returns nullptr.
|
|
LLVM_ABI MDNode *
|
|
getExplicitlyUnknownBranchWeightsIfProfiled(Function &F, StringRef PassName);
|
|
|
|
/// Analogous to setExplicitlyUnknownBranchWeights, but for functions and their
|
|
/// entry counts.
|
|
LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F,
|
|
StringRef PassName);
|
|
|
|
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD);
|
|
LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I);
|
|
|
|
/// Scaling the profile data attached to 'I' using the ratio of S/T.
|
|
LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
|
|
|
|
// Helper to apply a metadata setting function to an Instruction* if profiling
|
|
// is enabled. If profiling is disabled (ProfcheckDisableMetadataFixes is true)
|
|
// or V is not an Instruction, the callback will not be invoked.
|
|
LLVM_ABI void applyProfMetadataIfEnabled(
|
|
Value *V, llvm::function_ref<void(Instruction *)> setMetadataCallback);
|
|
|
|
/// Get the branch weights of a branch conditioned on b1 || b2, where b1 and b2
|
|
/// are 2 booleans that are the conditions of 2 branches for which we have the
|
|
/// branch weights B1 and B2, respectively. In both B1 and B2, the first
|
|
/// position (index 0) is for the 'true' branch, and the second position (index
|
|
/// 1) is for the 'false' branch.
|
|
template <typename T1, typename T2,
|
|
typename = typename std::enable_if<
|
|
std::is_arithmetic_v<T1> && std::is_arithmetic_v<T2> &&
|
|
sizeof(T1) <= sizeof(uint64_t) && sizeof(T2) <= sizeof(uint64_t)>>
|
|
inline SmallVector<uint64_t, 2>
|
|
getDisjunctionWeights(const SmallVector<T1, 2> &B1,
|
|
const SmallVector<T2, 2> &B2) {
|
|
// For the first conditional branch, the probability the "true" case is taken
|
|
// is p(b1) = B1[0] / (B1[0] + B1[1]). The "false" case's probability is
|
|
// p(not b1) = B1[1] / (B1[0] + B1[1]).
|
|
// Similarly for the second conditional branch and B2.
|
|
//
|
|
// The probability of the new branch NOT being taken is:
|
|
// not P = p((not b1) and (not b2)) =
|
|
// = B1[1] / (B1[0]+B1[1]) * B2[1] / (B2[0]+B2[1]) =
|
|
// = B1[1] * B2[1] / (B1[0] + B1[1]) * (B2[0] + B2[1])
|
|
// Then the probability of it being taken is: P = 1 - (not P).
|
|
// The denominator will be the same as above, and the numerator of P will be:
|
|
// (B1[0] + B1[1]) * (B2[0] + B2[1]) - B1[1]*B2[1]
|
|
// Which then reduces to what's shown below (out of the 4 terms coming out of
|
|
// the product of sums, the subtracted one cancels out).
|
|
assert(B1.size() == 2);
|
|
assert(B2.size() == 2);
|
|
|
|
uint64_t FalseWeight, TrueWeight;
|
|
|
|
if (!ProfcheckDisableMetadataFixes) {
|
|
FalseWeight = static_cast<uint64_t>(B1[1]) * B2[1];
|
|
TrueWeight =
|
|
static_cast<uint64_t>(B1[0]) * (static_cast<uint64_t>(B2[0]) + B2[1]) +
|
|
static_cast<uint64_t>(B1[1]) * B2[0];
|
|
} else {
|
|
FalseWeight = B1[1] * B2[1];
|
|
TrueWeight = B1[0] * (B2[0] + B2[1]) + B1[1] * B2[0];
|
|
}
|
|
return {TrueWeight, FalseWeight};
|
|
}
|
|
} // namespace llvm
|
|
#endif
|