//===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// This file provides the interface for the pseudo probe implementation for
/// AutoFDO.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
#define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
#include "llvm/ADT/DenseMap.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/PassManager.h"
#include "llvm/ProfileData/SampleProf.h"
#include <unordered_map>
namespace llvm {
class Any;
class BasicBlock;
class Function;
class Instruction;
class Loop;
class PassInstrumentationCallbacks;
class TargetMachine;
class Module;
using namespace sampleprof;
using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>;
using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>;
// Map from tuples of Probe id and inline stack hash code to distribution
// factors.
using ProbeFactorMap = std::unordered_map<std::pair<uint64_t, uint64_t>, float,
pair_hash<uint64_t, uint64_t>>;
using FuncProbeFactorMap = StringMap<ProbeFactorMap>;
class PseudoProbeDescriptor {
uint64_t FunctionGUID;
uint64_t FunctionHash;
public:
PseudoProbeDescriptor(uint64_t GUID, uint64_t Hash)
: FunctionGUID(GUID), FunctionHash(Hash) {}
uint64_t getFunctionGUID() const { return FunctionGUID; }
uint64_t getFunctionHash() const { return FunctionHash; }
};
// A pseudo probe verifier that can be run after each IR passes to detect the
// violation of updating probe factors. In principle, the sum of distribution
// factor for a probe should be identical before and after a pass. For a
// function pass, the factor sum for a probe would be typically 100%.
class PseudoProbeVerifier {
public:
void registerCallbacks(PassInstrumentationCallbacks &PIC);
// Implementation of pass instrumentation callbacks for new pass manager.
void runAfterPass(StringRef PassID, Any IR);
private:
// Allow a little bias due the rounding to integral factors.
constexpr static float DistributionFactorVariance = 0.02f;
// Distribution factors from last pass.
FuncProbeFactorMap FunctionProbeFactors;
void collectProbeFactors(const BasicBlock *BB, ProbeFactorMap &ProbeFactors);
void runAfterPass(const Module *M);
void runAfterPass(const LazyCallGraph::SCC *C);
void runAfterPass(const Function *F);
void runAfterPass(const Loop *L);
bool shouldVerifyFunction(const Function *F);
void verifyProbeFactors(const Function *F,
const ProbeFactorMap &ProbeFactors);
};
// This class serves sample counts correlation for SampleProfileLoader by
// analyzing pseudo probes and their function descriptors injected by
// SampleProfileProber.
class PseudoProbeManager {
DenseMap<uint64_t, PseudoProbeDescriptor> GUIDToProbeDescMap;
const PseudoProbeDescriptor *getDesc(const Function &F) const;
public:
PseudoProbeManager(const Module &M);
bool moduleIsProbed(const Module &M) const;
bool profileIsValid(const Function &F, const FunctionSamples &Samples) const;
};
/// Sample profile pseudo prober.
///
/// Insert pseudo probes for block sampling and value sampling.
class SampleProfileProber {
public:
// Give an empty module id when the prober is not used for instrumentation.
SampleProfileProber(Function &F, const std::string &CurModuleUniqueId);
void instrumentOneFunc(Function &F, TargetMachine *TM);
private:
Function *getFunction() const { return F; }
uint64_t getFunctionHash() const { return FunctionHash; }
uint32_t getBlockId(const BasicBlock *BB) const;
uint32_t getCallsiteId(const Instruction *Call) const;
void computeCFGHash();
void computeProbeIdForBlocks();
void computeProbeIdForCallsites();
Function *F;
/// The current module ID that is used to name a static object as a comdat
/// group.
std::string CurModuleUniqueId;
/// A CFG hash code used to identify a function code changes.
uint64_t FunctionHash;
/// Map basic blocks to the their pseudo probe ids.
BlockIdMap BlockProbeIds;
/// Map indirect calls to the their pseudo probe ids.
InstructionIdMap CallProbeIds;
/// The ID of the last probe, Can be used to number a new probe.
uint32_t LastProbeId;
};
class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> {
TargetMachine *TM;
public:
SampleProfileProbePass(TargetMachine *TM) : TM(TM) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
};
// Pseudo probe distribution factor updater.
// Sample profile annotation can happen in both LTO prelink and postlink. The
// postlink-time re-annotation can degrade profile quality because of prelink
// code duplication transformation, such as loop unrolling, jump threading,
// indirect call promotion etc. As such, samples corresponding to a source
// location may be aggregated multiple times in postlink. With a concept of
// distribution factor for pseudo probes, samples can be distributed among
// duplicated probes reasonable based on the assumption that optimizations
// duplicating code well-maintain the branch frequency information (BFI). This
// pass updates distribution factors for each pseudo probe at the end of the
// prelink pipeline, to reflect an estimated portion of the real execution
// count.
class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> {
void runOnFunction(Function &F, FunctionAnalysisManager &FAM);
public:
PseudoProbeUpdatePass() = default;
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
};
} // end namespace llvm
#endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H