Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file contains common definitions used in the reading and writing of

// sample profile data.

//

//===----------------------------------------------------------------------===//

#ifndef LLVM_PROFILEDATA_SAMPLEPROF_H

#define LLVM_PROFILEDATA_SAMPLEPROF_H

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/StringExtras.h"

#include "llvm/ADT/StringMap.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/GlobalValue.h"

#include "llvm/Support/Allocator.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/ErrorOr.h"

#include "llvm/Support/MathExtras.h"

#include <algorithm>

#include <cstdint>

#include <list>

#include <map>

#include <set>

#include <sstream>

#include <string>

#include <system_error>

#include <unordered_map>

#include <utility>

namespace llvm {

class DILocation;

class raw_ostream;

const std::error_category &sampleprof_category();

enum class sampleprof_error {

  success = 0,

  bad_magic,

  unsupported_version,

  too_large,

  truncated,

  malformed,

  unrecognized_format,

  unsupported_writing_format,

  truncated_name_table,

  not_implemented,

  counter_overflow,

  ostream_seek_unsupported,

  uncompress_failed,

  zlib_unavailable,

  hash_mismatch

};

inline std::error_code make_error_code(sampleprof_error E) {

  return std::error_code(static_cast<int>(E), sampleprof_category());

}

inline sampleprof_error MergeResult(sampleprof_error &Accumulator,

                                    sampleprof_error Result) {

  // Prefer first error encountered as later errors may be secondary effects of

  // the initial problem.

  if (Accumulator == sampleprof_error::success &&

      Result != sampleprof_error::success)

    Accumulator = Result;

  return Accumulator;

}

} // end namespace llvm

namespace std {

template <>

struct is_error_code_enum<llvm::sampleprof_error> : std::true_type {};

} // end namespace std

namespace llvm {

namespace sampleprof {

enum SampleProfileFormat {

  SPF_None = 0,

  SPF_Text = 0x1,

  SPF_Compact_Binary = 0x2,

  SPF_GCC = 0x3,

  SPF_Ext_Binary = 0x4,

  SPF_Binary = 0xff

};

static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {

  return uint64_t('S') << (64 - 8) | uint64_t('P') << (64 - 16) |

         uint64_t('R') << (64 - 24) | uint64_t('O') << (64 - 32) |

         uint64_t('F') << (64 - 40) | uint64_t('4') << (64 - 48) |

         uint64_t('2') << (64 - 56) | uint64_t(Format);

}

/// Get the proper representation of a string according to whether the

/// current Format uses MD5 to represent the string.

static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,

                                       std::string &GUIDBuf) {

  if (Name.empty() || !UseMD5)

    return Name;

  GUIDBuf = std::to_string(Function::getGUID(Name));

  return GUIDBuf;

}

static inline uint64_t SPVersion() { return 103; }

// Section Type used by SampleProfileExtBinaryBaseReader and

// SampleProfileExtBinaryBaseWriter. Never change the existing

// value of enum. Only append new ones.

enum SecType {

  SecInValid = 0,

  SecProfSummary = 1,

  SecNameTable = 2,

  SecProfileSymbolList = 3,

  SecFuncOffsetTable = 4,

  SecFuncMetadata = 5,

  SecCSNameTable = 6,

  // marker for the first type of profile.

  SecFuncProfileFirst = 32,

  SecLBRProfile = SecFuncProfileFirst

};

static inline std::string getSecName(SecType Type) {

  switch ((int)Type) { // Avoid -Wcovered-switch-default

  case SecInValid:

    return "InvalidSection";

  case SecProfSummary:

    return "ProfileSummarySection";

  case SecNameTable:

    return "NameTableSection";

  case SecProfileSymbolList:

    return "ProfileSymbolListSection";

  case SecFuncOffsetTable:

    return "FuncOffsetTableSection";

  case SecFuncMetadata:

    return "FunctionMetadata";

  case SecCSNameTable:

    return "CSNameTableSection";

  case SecLBRProfile:

    return "LBRProfileSection";

  default:

    return "UnknownSection";

  }

}

// Entry type of section header table used by SampleProfileExtBinaryBaseReader

// and SampleProfileExtBinaryBaseWriter.

struct SecHdrTableEntry {

  SecType Type;

  uint64_t Flags;

  uint64_t Offset;

  uint64_t Size;

  // The index indicating the location of the current entry in

  // SectionHdrLayout table.

  uint32_t LayoutIndex;

};

// Flags common for all sections are defined here. In SecHdrTableEntry::Flags,

// common flags will be saved in the lower 32bits and section specific flags

// will be saved in the higher 32 bits.

enum class SecCommonFlags : uint32_t {

  SecFlagInValid = 0,

  SecFlagCompress = (1 << 0),

  // Indicate the section contains only profile without context.

  SecFlagFlat = (1 << 1)

};

// Section specific flags are defined here.

// !!!Note: Everytime a new enum class is created here, please add

// a new check in verifySecFlag.

enum class SecNameTableFlags : uint32_t {

  SecFlagInValid = 0,

  SecFlagMD5Name = (1 << 0),

  // Store MD5 in fixed length instead of ULEB128 so NameTable can be

  // accessed like an array.

  SecFlagFixedLengthMD5 = (1 << 1),

  // Profile contains ".__uniq." suffix name. Compiler shouldn't strip

  // the suffix when doing profile matching when seeing the flag.

  SecFlagUniqSuffix = (1 << 2)

};

enum class SecProfSummaryFlags : uint32_t {

  SecFlagInValid = 0,

  /// SecFlagPartial means the profile is for common/shared code.

  /// The common profile is usually merged from profiles collected

  /// from running other targets.

  SecFlagPartial = (1 << 0),

  /// SecFlagContext means this is context-sensitive flat profile for

  /// CSSPGO

  SecFlagFullContext = (1 << 1),

  /// SecFlagFSDiscriminator means this profile uses flow-sensitive

  /// discriminators.

  SecFlagFSDiscriminator = (1 << 2),

  /// SecFlagIsPreInlined means this profile contains ShouldBeInlined

  /// contexts thus this is CS preinliner computed.

  SecFlagIsPreInlined = (1 << 4),

};

enum class SecFuncMetadataFlags : uint32_t {

  SecFlagInvalid = 0,

  SecFlagIsProbeBased = (1 << 0),

  SecFlagHasAttribute = (1 << 1),

};

enum class SecFuncOffsetFlags : uint32_t {

  SecFlagInvalid = 0,

  // Store function offsets in an order of contexts. The order ensures that

  // callee contexts of a given context laid out next to it.

  SecFlagOrdered = (1 << 0),

};

// Verify section specific flag is used for the correct section.

template <class SecFlagType>

static inline void verifySecFlag(SecType Type, SecFlagType Flag) {

  // No verification is needed for common flags.

  if (std::is_same<SecCommonFlags, SecFlagType>())

    return;

  // Verification starts here for section specific flag.

  bool IsFlagLegal = false;

  switch (Type) {

  case SecNameTable:

    IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();

    break;

  case SecProfSummary:

    IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();

    break;

  case SecFuncMetadata:

    IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>();

    break;

  default:

  case SecFuncOffsetTable:

    IsFlagLegal = std::is_same<SecFuncOffsetFlags, SecFlagType>();

    break;

  }

  if (!IsFlagLegal)

    llvm_unreachable("Misuse of a flag in an incompatible section");

}

template <class SecFlagType>

static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {

  verifySecFlag(Entry.Type, Flag);

  auto FVal = static_cast<uint64_t>(Flag);

  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();

  Entry.Flags |= IsCommon ? FVal : (FVal << 32);

}

template <class SecFlagType>

static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {

  verifySecFlag(Entry.Type, Flag);

  auto FVal = static_cast<uint64_t>(Flag);

  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();

  Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));

}

template <class SecFlagType>

static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {

  verifySecFlag(Entry.Type, Flag);

  auto FVal = static_cast<uint64_t>(Flag);

  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();

  return Entry.Flags & (IsCommon ? FVal : (FVal << 32));

}

/// Represents the relative location of an instruction.

///

/// Instruction locations are specified by the line offset from the

/// beginning of the function (marked by the line where the function

/// header is) and the discriminator value within that line.

///

/// The discriminator value is useful to distinguish instructions

/// that are on the same line but belong to different basic blocks

/// (e.g., the two post-increment instructions in "if (p) x++; else y++;").

struct LineLocation {

  LineLocation(uint32_t L, uint32_t D) : LineOffset(L), Discriminator(D) {}

  void print(raw_ostream &OS) const;

  void dump() const;

  bool operator<(const LineLocation &O) const {

    return LineOffset < O.LineOffset ||

           (LineOffset == O.LineOffset && Discriminator < O.Discriminator);

  }

  bool operator==(const LineLocation &O) const {

    return LineOffset == O.LineOffset && Discriminator == O.Discriminator;

  }

  bool operator!=(const LineLocation &O) const {

    return LineOffset != O.LineOffset || Discriminator != O.Discriminator;

  }

  uint32_t LineOffset;

  uint32_t Discriminator;

};

struct LineLocationHash {

  uint64_t operator()(const LineLocation &Loc) const {

    return std::hash<std::uint64_t>{}((((uint64_t)Loc.LineOffset) << 32) |

                                      Loc.Discriminator);

  }

};

raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);

/// Representation of a single sample record.

///

/// A sample record is represented by a positive integer value, which

/// indicates how frequently was the associated line location executed.

///

/// Additionally, if the associated location contains a function call,

/// the record will hold a list of all the possible called targets. For

/// direct calls, this will be the exact function being invoked. For

/// indirect calls (function pointers, virtual table dispatch), this

/// will be a list of one or more functions.

class SampleRecord {

public:

  using CallTarget = std::pair<StringRef, uint64_t>;

  struct CallTargetComparator {

    bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {

      if (LHS.second != RHS.second)

        return LHS.second > RHS.second;

      return LHS.first < RHS.first;

    }

  };

  using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;

  using CallTargetMap = StringMap<uint64_t>;

  SampleRecord() = default;

  /// Increment the number of samples for this record by \p S.

  /// Optionally scale sample count \p S by \p Weight.

  ///

  /// Sample counts accumulate using saturating arithmetic, to avoid wrapping

  /// around unsigned integers.

  sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) {

    bool Overflowed;

    NumSamples = SaturatingMultiplyAdd(S, Weight, NumSamples, &Overflowed);

    return Overflowed ? sampleprof_error::counter_overflow

                      : sampleprof_error::success;

  }

  /// Decrease the number of samples for this record by \p S. Return the amout

  /// of samples actually decreased.

  uint64_t removeSamples(uint64_t S) {

    if (S > NumSamples)

      S = NumSamples;

    NumSamples -= S;

    return S;

  }

  /// Add called function \p F with samples \p S.

  /// Optionally scale sample count \p S by \p Weight.

  ///

  /// Sample counts accumulate using saturating arithmetic, to avoid wrapping

  /// around unsigned integers.

  sampleprof_error addCalledTarget(StringRef F, uint64_t S,

                                   uint64_t Weight = 1) {

    uint64_t &TargetSamples = CallTargets[F];

    bool Overflowed;

    TargetSamples =

        SaturatingMultiplyAdd(S, Weight, TargetSamples, &Overflowed);

    return Overflowed ? sampleprof_error::counter_overflow

                      : sampleprof_error::success;

  }

  /// Remove called function from the call target map. Return the target sample

  /// count of the called function.

  uint64_t removeCalledTarget(StringRef F) {

    uint64_t Count = 0;

    auto I = CallTargets.find(F);

    if (I != CallTargets.end()) {

      Count = I->second;

      CallTargets.erase(I);

    }

    return Count;

  }

  /// Return true if this sample record contains function calls.

  bool hasCalls() const { return !CallTargets.empty(); }

  uint64_t getSamples() const { return NumSamples; }

  const CallTargetMap &getCallTargets() const { return CallTargets; }

  const SortedCallTargetSet getSortedCallTargets() const {

    return SortCallTargets(CallTargets);

  }

  uint64_t getCallTargetSum() const {

    uint64_t Sum = 0;

    for (const auto &I : CallTargets)

      Sum += I.second;

    return Sum;

  }

  /// Sort call targets in descending order of call frequency.

  static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) {

    SortedCallTargetSet SortedTargets;

    for (const auto &[Target, Frequency] : Targets) {

      SortedTargets.emplace(Target, Frequency);

    }

    return SortedTargets;

  }

  /// Prorate call targets by a distribution factor.

  static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets,

                                               float DistributionFactor) {

    CallTargetMap AdjustedTargets;

    for (const auto &[Target, Frequency] : Targets) {

      AdjustedTargets[Target] = Frequency * DistributionFactor;

    }

    return AdjustedTargets;

  }

  /// Merge the samples in \p Other into this record.

  /// Optionally scale sample counts by \p Weight.

  sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1);

  void print(raw_ostream &OS, unsigned Indent) const;

  void dump() const;

private:

  uint64_t NumSamples = 0;

  CallTargetMap CallTargets;

};

raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);

// State of context associated with FunctionSamples

enum ContextStateMask {

  UnknownContext = 0x0,   // Profile without context

  RawContext = 0x1,       // Full context profile from input profile

  SyntheticContext = 0x2, // Synthetic context created for context promotion

  InlinedContext = 0x4,   // Profile for context that is inlined into caller

  MergedContext = 0x8     // Profile for context merged into base profile

};

// Attribute of context associated with FunctionSamples

enum ContextAttributeMask {

  ContextNone = 0x0,

  ContextWasInlined = 0x1,      // Leaf of context was inlined in previous build

  ContextShouldBeInlined = 0x2, // Leaf of context should be inlined

  ContextDuplicatedIntoBase =

      0x4, // Leaf of context is duplicated into the base profile

};

// Represents a context frame with function name and line location

struct SampleContextFrame {

  StringRef FuncName;

  LineLocation Location;

  SampleContextFrame() : Location(0, 0) {}

  SampleContextFrame(StringRef FuncName, LineLocation Location)

      : FuncName(FuncName), Location(Location) {}

  bool operator==(const SampleContextFrame &That) const {

    return Location == That.Location && FuncName == That.FuncName;

  }

  bool operator!=(const SampleContextFrame &That) const {

    return !(*this == That);

  }

  std::string toString(bool OutputLineLocation) const {

    std::ostringstream OContextStr;

    OContextStr << FuncName.str();

    if (OutputLineLocation) {

      OContextStr << ":" << Location.LineOffset;

      if (Location.Discriminator)

        OContextStr << "." << Location.Discriminator;

    }

    return OContextStr.str();

  }

};

static inline hash_code hash_value(const SampleContextFrame &arg) {

  return hash_combine(arg.FuncName, arg.Location.LineOffset,

                      arg.Location.Discriminator);

}

using SampleContextFrameVector = SmallVector<SampleContextFrame, 1>;

using SampleContextFrames = ArrayRef<SampleContextFrame>;

struct SampleContextFrameHash {

  uint64_t operator()(const SampleContextFrameVector &S) const {

    return hash_combine_range(S.begin(), S.end());

  }

};

// Sample context for FunctionSamples. It consists of the calling context,

// the function name and context state. Internally sample context is represented

// using ArrayRef, which is also the input for constructing a `SampleContext`.

// It can accept and represent both full context string as well as context-less

// function name.

// For a CS profile, a full context vector can look like:

//    `main:3 _Z5funcAi:1 _Z8funcLeafi`

// For a base CS profile without calling context, the context vector should only

// contain the leaf frame name.

// For a non-CS profile, the context vector should be empty.

class SampleContext {

public:

  SampleContext() : State(UnknownContext), Attributes(ContextNone) {}

  SampleContext(StringRef Name)

      : Name(Name), State(UnknownContext), Attributes(ContextNone) {}

  SampleContext(SampleContextFrames Context,

                ContextStateMask CState = RawContext)

      : Attributes(ContextNone) {

    assert(!Context.empty() && "Context is empty");

    setContext(Context, CState);

  }

  // Give a context string, decode and populate internal states like

  // Function name, Calling context and context state. Example of input

  // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`

  SampleContext(StringRef ContextStr,

                std::list<SampleContextFrameVector> &CSNameTable,

                ContextStateMask CState = RawContext)

      : Attributes(ContextNone) {

    assert(!ContextStr.empty());

    // Note that `[]` wrapped input indicates a full context string, otherwise

    // it's treated as context-less function name only.

    bool HasContext = ContextStr.startswith("[");

    if (!HasContext) {

      State = UnknownContext;

      Name = ContextStr;

    } else {

      CSNameTable.emplace_back();

      SampleContextFrameVector &Context = CSNameTable.back();

      createCtxVectorFromStr(ContextStr, Context);

      setContext(Context, CState);

    }

  }

  /// Create a context vector from a given context string and save it in

  /// `Context`.

  static void createCtxVectorFromStr(StringRef ContextStr,

                                     SampleContextFrameVector &Context) {

    // Remove encapsulating '[' and ']' if any

    ContextStr = ContextStr.substr(1, ContextStr.size() - 2);

    StringRef ContextRemain = ContextStr;

    StringRef ChildContext;

    StringRef CalleeName;

    while (!ContextRemain.empty()) {

      auto ContextSplit = ContextRemain.split(" @ ");

      ChildContext = ContextSplit.first;

      ContextRemain = ContextSplit.second;

      LineLocation CallSiteLoc(0, 0);

      decodeContextString(ChildContext, CalleeName, CallSiteLoc);

      Context.emplace_back(CalleeName, CallSiteLoc);

    }

  }

  // Decode context string for a frame to get function name and location.

  // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.

  static void decodeContextString(StringRef ContextStr, StringRef &FName,

                                  LineLocation &LineLoc) {

    // Get function name

    auto EntrySplit = ContextStr.split(':');

    FName = EntrySplit.first;

    LineLoc = {0, 0};

    if (!EntrySplit.second.empty()) {

      // Get line offset, use signed int for getAsInteger so string will

      // be parsed as signed.

      int LineOffset = 0;

      auto LocSplit = EntrySplit.second.split('.');

      LocSplit.first.getAsInteger(10, LineOffset);

      LineLoc.LineOffset = LineOffset;

      // Get discriminator

      if (!LocSplit.second.empty())

        LocSplit.second.getAsInteger(10, LineLoc.Discriminator);

    }

  }

  operator SampleContextFrames() const { return FullContext; }

  bool hasAttribute(ContextAttributeMask A) { return Attributes & (uint32_t)A; }

  void setAttribute(ContextAttributeMask A) { Attributes |= (uint32_t)A; }

  uint32_t getAllAttributes() { return Attributes; }

  void setAllAttributes(uint32_t A) { Attributes = A; }

  bool hasState(ContextStateMask S) { return State & (uint32_t)S; }

  void setState(ContextStateMask S) { State |= (uint32_t)S; }

  void clearState(ContextStateMask S) { State &= (uint32_t)~S; }

  bool hasContext() const { return State != UnknownContext; }

  bool isBaseContext() const { return FullContext.size() == 1; }

  StringRef getName() const { return Name; }

  SampleContextFrames getContextFrames() const { return FullContext; }

  static std::string getContextString(SampleContextFrames Context,

                                      bool IncludeLeafLineLocation = false) {

    std::ostringstream OContextStr;

    for (uint32_t I = 0; I < Context.size(); I++) {

      if (OContextStr.str().size()) {

        OContextStr << " @ ";

      }

      OContextStr << Context[I].toString(I != Context.size() - 1 ||

                                         IncludeLeafLineLocation);

    }

    return OContextStr.str();

  }

  std::string toString() const {

    if (!hasContext())

      return Name.str();

    return getContextString(FullContext, false);

  }

  uint64_t getHashCode() const {

    return hasContext() ? hash_value(getContextFrames())

                        : hash_value(getName());

  }

  /// Set the name of the function and clear the current context.

  void setName(StringRef FunctionName) {

    Name = FunctionName;

    FullContext = SampleContextFrames();

    State = UnknownContext;

  }

  void setContext(SampleContextFrames Context,

                  ContextStateMask CState = RawContext) {

    assert(CState != UnknownContext);

    FullContext = Context;

    Name = Context.back().FuncName;

    State = CState;

  }

  bool operator==(const SampleContext &That) const {

    return State == That.State && Name == That.Name &&

           FullContext == That.FullContext;

  }

  bool operator!=(const SampleContext &That) const { return !(*this == That); }

  bool operator<(const SampleContext &That) const {

    if (State != That.State)

      return State < That.State;

    if (!hasContext()) {

      return Name < That.Name;

    }

    uint64_t I = 0;

    while (I < std::min(FullContext.size(), That.FullContext.size())) {

      auto &Context1 = FullContext[I];

      auto &Context2 = That.FullContext[I];

      auto V = Context1.FuncName.compare(Context2.FuncName);

      if (V)

        return V < 0;

      if (Context1.Location != Context2.Location)

        return Context1.Location < Context2.Location;

      I++;

    }

    return FullContext.size() < That.FullContext.size();

  }

  struct Hash {

    uint64_t operator()(const SampleContext &Context) const {

      return Context.getHashCode();

    }

  };

  bool IsPrefixOf(const SampleContext &That) const {

    auto ThisContext = FullContext;

    auto ThatContext = That.FullContext;

    if (ThatContext.size() < ThisContext.size())

      return false;

    ThatContext = ThatContext.take_front(ThisContext.size());

    // Compare Leaf frame first

    if (ThisContext.back().FuncName != ThatContext.back().FuncName)

      return false;

    // Compare leading context

    return ThisContext.drop_back() == ThatContext.drop_back();

  }

private:

  /// Mangled name of the function.

  StringRef Name;

  // Full context including calling context and leaf function name

  SampleContextFrames FullContext;

  // State of the associated sample profile

  uint32_t State;

  // Attribute of the associated sample profile

  uint32_t Attributes;

};

static inline hash_code hash_value(const SampleContext &arg) {

  return arg.hasContext() ? hash_value(arg.getContextFrames())

                          : hash_value(arg.getName());

}

class FunctionSamples;

class SampleProfileReaderItaniumRemapper;

using BodySampleMap = std::map<LineLocation, SampleRecord>;

// NOTE: Using a StringMap here makes parsed profiles consume around 17% more

// memory, which is *very* significant for large profiles.

using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;

using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;

/// Representation of the samples collected for a function.

///

/// This data structure contains all the collected samples for the body

/// of a function. Each sample corresponds to a LineLocation instance

/// within the body of the function.

class FunctionSamples {

public:

  FunctionSamples() = default;

  void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;

  void dump() const;

  sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) {

    bool Overflowed;

    TotalSamples =

        SaturatingMultiplyAdd(Num, Weight, TotalSamples, &Overflowed);

    return Overflowed ? sampleprof_error::counter_overflow

                      : sampleprof_error::success;

  }

  void removeTotalSamples(uint64_t Num) {

    if (TotalSamples < Num)

      TotalSamples = 0;

    else

      TotalSamples -= Num;

  }

  void setTotalSamples(uint64_t Num) { TotalSamples = Num; }

  sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {

    bool Overflowed;

    TotalHeadSamples =

        SaturatingMultiplyAdd(Num, Weight, TotalHeadSamples, &Overflowed);

    return Overflowed ? sampleprof_error::counter_overflow

                      : sampleprof_error::success;

  }

  sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator,

                                  uint64_t Num, uint64_t Weight = 1) {

    return BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(

        Num, Weight);

  }

  sampleprof_error addCalledTargetSamples(uint32_t LineOffset,

                                          uint32_t Discriminator,

                                          StringRef FName, uint64_t Num,

                                          uint64_t Weight = 1) {

    return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(

        FName, Num, Weight);

  }

  // Remove a call target and decrease the body sample correspondingly. Return

  // the number of body samples actually decreased.

  uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,

                                           uint32_t Discriminator,

                                           StringRef FName) {

    uint64_t Count = 0;

    auto I = BodySamples.find(LineLocation(LineOffset, Discriminator));

    if (I != BodySamples.end()) {

      Count = I->second.removeCalledTarget(FName);

      Count = I->second.removeSamples(Count);

      if (!I->second.getSamples())

        BodySamples.erase(I);

    }

    return Count;

  }

  sampleprof_error addBodySamplesForProbe(uint32_t Index, uint64_t Num,

                                          uint64_t Weight = 1) {

    SampleRecord S;

    S.addSamples(Num, Weight);