Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// Instrumentation-based profiling data is generated by instrumented

// binaries through library functions in compiler-rt, and read by the clang

// frontend to feed PGO.

//

//===----------------------------------------------------------------------===//

#ifndef LLVM_PROFILEDATA_INSTRPROF_H

#define LLVM_PROFILEDATA_INSTRPROF_H

#include "llvm/ADT/ArrayRef.h"

#include "llvm/ADT/BitmaskEnum.h"

#include "llvm/ADT/STLExtras.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/ADT/StringSet.h"

#include "llvm/ADT/Triple.h"

#include "llvm/IR/GlobalValue.h"

#include "llvm/IR/ProfileSummary.h"

#include "llvm/ProfileData/InstrProfData.inc"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Compiler.h"

#include "llvm/Support/Endian.h"

#include "llvm/Support/Error.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/Host.h"

#include "llvm/Support/MD5.h"

#include "llvm/Support/MathExtras.h"

#include "llvm/Support/raw_ostream.h"

#include <algorithm>

#include <cassert>

#include <cstddef>

#include <cstdint>

#include <cstring>

#include <list>

#include <memory>

#include <string>

#include <system_error>

#include <utility>

#include <vector>

namespace llvm {

class Function;

class GlobalVariable;

struct InstrProfRecord;

class InstrProfSymtab;

class Instruction;

class MDNode;

class Module;

enum InstrProfSectKind {

#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,

#include "llvm/ProfileData/InstrProfData.inc"

};

/// Return the max count value. We reserver a few large values for special use.

inline uint64_t getInstrMaxCountValue() {

  return std::numeric_limits<uint64_t>::max() - 2;

}

/// Return the name of the profile section corresponding to \p IPSK.

///

/// The name of the section depends on the object format type \p OF. If

/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may

/// be added to the section name (this is the default).

std::string getInstrProfSectionName(InstrProfSectKind IPSK,

                                    Triple::ObjectFormatType OF,

                                    bool AddSegmentInfo = true);

/// Return the name profile runtime entry point to do value profiling

/// for a given site.

inline StringRef getInstrProfValueProfFuncName() {

  return INSTR_PROF_VALUE_PROF_FUNC_STR;

}

/// Return the name profile runtime entry point to do memop size value

/// profiling.

inline StringRef getInstrProfValueProfMemOpFuncName() {

  return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;

}

/// Return the name prefix of variables containing instrumented function names.

inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }

/// Return the name prefix of variables containing per-function control data.

inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }

/// Return the name prefix of profile counter variables.

inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }

/// Return the name prefix of value profile variables.

inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }

/// Return the name of value profile node array variables:

inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }

/// Return the name of the variable holding the strings (possibly compressed)

/// of all function's PGO names.

inline StringRef getInstrProfNamesVarName() {

  return "__llvm_prf_nm";

}

/// Return the name of a covarage mapping variable (internal linkage)

/// for each instrumented source module. Such variables are allocated

/// in the __llvm_covmap section.

inline StringRef getCoverageMappingVarName() {

  return "__llvm_coverage_mapping";

}

/// Return the name of the internal variable recording the array

/// of PGO name vars referenced by the coverage mapping. The owning

/// functions of those names are not emitted by FE (e.g, unused inline

/// functions.)

inline StringRef getCoverageUnusedNamesVarName() {

  return "__llvm_coverage_names";

}

/// Return the name of function that registers all the per-function control

/// data at program startup time by calling __llvm_register_function. This

/// function has internal linkage and is called by  __llvm_profile_init

/// runtime method. This function is not generated for these platforms:

/// Darwin, Linux, and FreeBSD.

inline StringRef getInstrProfRegFuncsName() {

  return "__llvm_profile_register_functions";

}

/// Return the name of the runtime interface that registers per-function control

/// data for one instrumented function.

inline StringRef getInstrProfRegFuncName() {

  return "__llvm_profile_register_function";

}

/// Return the name of the runtime interface that registers the PGO name strings.

inline StringRef getInstrProfNamesRegFuncName() {

  return "__llvm_profile_register_names_function";

}

/// Return the name of the runtime initialization method that is generated by

/// the compiler. The function calls __llvm_profile_register_functions and

/// __llvm_profile_override_default_filename functions if needed. This function

/// has internal linkage and invoked at startup time via init_array.

inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }

/// Return the name of the hook variable defined in profile runtime library.

/// A reference to the variable causes the linker to link in the runtime

/// initialization module (which defines the hook variable).

inline StringRef getInstrProfRuntimeHookVarName() {

  return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);

}

/// Return the name of the compiler generated function that references the

/// runtime hook variable. The function is a weak global.

inline StringRef getInstrProfRuntimeHookVarUseFuncName() {

  return "__llvm_profile_runtime_user";

}

inline StringRef getInstrProfCounterBiasVarName() {

  return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);

}

/// Return the marker used to separate PGO names during serialization.

inline StringRef getInstrProfNameSeparator() { return "\01"; }

/// Return the modified name for function \c F suitable to be

/// used the key for profile lookup. Variable \c InLTO indicates if this

/// is called in LTO optimization passes.

std::string getPGOFuncName(const Function &F, bool InLTO = false,

                           uint64_t Version = INSTR_PROF_INDEX_VERSION);

/// Return the modified name for a function suitable to be

/// used the key for profile lookup. The function's original

/// name is \c RawFuncName and has linkage of type \c Linkage.

/// The function is defined in module \c FileName.

std::string getPGOFuncName(StringRef RawFuncName,

                           GlobalValue::LinkageTypes Linkage,

                           StringRef FileName,

                           uint64_t Version = INSTR_PROF_INDEX_VERSION);

/// Return the name of the global variable used to store a function

/// name in PGO instrumentation. \c FuncName is the name of the function

/// returned by the \c getPGOFuncName call.

std::string getPGOFuncNameVarName(StringRef FuncName,

                                  GlobalValue::LinkageTypes Linkage);

/// Create and return the global variable for function name used in PGO

/// instrumentation. \c FuncName is the name of the function returned

/// by \c getPGOFuncName call.

GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);

/// Create and return the global variable for function name used in PGO

/// instrumentation.  /// \c FuncName is the name of the function

/// returned by \c getPGOFuncName call, \c M is the owning module,

/// and \c Linkage is the linkage of the instrumented function.

GlobalVariable *createPGOFuncNameVar(Module &M,

                                     GlobalValue::LinkageTypes Linkage,

                                     StringRef PGOFuncName);

/// Return the initializer in string of the PGO name var \c NameVar.

StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);

/// Given a PGO function name, remove the filename prefix and return

/// the original (static) function name.

StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,

                                   StringRef FileName = "<unknown>");

/// Given a vector of strings (function PGO names) \c NameStrs, the

/// method generates a combined string \c Result that is ready to be

/// serialized.  The \c Result string is comprised of three fields:

/// The first field is the length of the uncompressed strings, and the

/// the second field is the length of the zlib-compressed string.

/// Both fields are encoded in ULEB128.  If \c doCompress is false, the

///  third field is the uncompressed strings; otherwise it is the

/// compressed string. When the string compression is off, the

/// second field will have value zero.

Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,

                                bool doCompression, std::string &Result);

/// Produce \c Result string with the same format described above. The input

/// is vector of PGO function name variables that are referenced.

Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,

                                std::string &Result, bool doCompression = true);

/// \c NameStrings is a string composed of one of more sub-strings encoded in

/// the format described above. The substrings are separated by 0 or more zero

/// bytes. This method decodes the string and populates the \c Symtab.

Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);

/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being

/// set in IR PGO compilation.

bool isIRPGOFlagSet(const Module *M);

/// Check if we can safely rename this Comdat function. Instances of the same

/// comdat function may have different control flows thus can not share the

/// same counter variable.

bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);

enum InstrProfValueKind : uint32_t {

#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,

#include "llvm/ProfileData/InstrProfData.inc"

};

/// Get the value profile data for value site \p SiteIdx from \p InstrProfR

/// and annotate the instruction \p Inst with the value profile meta data.

/// Annotate up to \p MaxMDCount (default 3) number of records per value site.

void annotateValueSite(Module &M, Instruction &Inst,

                       const InstrProfRecord &InstrProfR,

                       InstrProfValueKind ValueKind, uint32_t SiteIndx,

                       uint32_t MaxMDCount = 3);

/// Same as the above interface but using an ArrayRef, as well as \p Sum.

void annotateValueSite(Module &M, Instruction &Inst,

                       ArrayRef<InstrProfValueData> VDs, uint64_t Sum,

                       InstrProfValueKind ValueKind, uint32_t MaxMDCount);

/// Extract the value profile data from \p Inst which is annotated with

/// value profile meta data. Return false if there is no value data annotated,

/// otherwise  return true.

bool getValueProfDataFromInst(const Instruction &Inst,

                              InstrProfValueKind ValueKind,

                              uint32_t MaxNumValueData,

                              InstrProfValueData ValueData[],

                              uint32_t &ActualNumValueData, uint64_t &TotalC,

                              bool GetNoICPValue = false);

inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }

/// Return the PGOFuncName meta data associated with a function.

MDNode *getPGOFuncNameMetadata(const Function &F);

/// Create the PGOFuncName meta data if PGOFuncName is different from

/// function's raw name. This should only apply to internal linkage functions

/// declared by users only.

void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);

/// Check if we can use Comdat for profile variables. This will eliminate

/// the duplicated profile variables for Comdat functions.

bool needsComdatForCounter(const Function &F, const Module &M);

/// An enum describing the attributes of an instrumented profile.

enum class InstrProfKind {

  Unknown = 0x0,

  // A frontend clang profile, incompatible with other attrs.

  FrontendInstrumentation = 0x1,

  // An IR-level profile (default when -fprofile-generate is used).

  IRInstrumentation = 0x2,

  // A profile with entry basic block instrumentation.

  FunctionEntryInstrumentation = 0x4,

  // A context sensitive IR-level profile.

  ContextSensitive = 0x8,

  // Use single byte probes for coverage.

  SingleByteCoverage = 0x10,

  // Only instrument the function entry basic block.

  FunctionEntryOnly = 0x20,

  // A memory profile collected using -fprofile=memory.

  MemProf = 0x40,

  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/MemProf)

};

const std::error_category &instrprof_category();

enum class instrprof_error {

  success = 0,

  eof,

  unrecognized_format,

  bad_magic,

  bad_header,

  unsupported_version,

  unsupported_hash_type,

  too_large,

  truncated,

  malformed,

  missing_debug_info_for_correlation,

  unexpected_debug_info_for_correlation,

  unable_to_correlate_profile,

  unknown_function,

  invalid_prof,

  hash_mismatch,

  count_mismatch,

  counter_overflow,

  value_site_count_mismatch,

  compress_failed,

  uncompress_failed,

  empty_raw_profile,

  zlib_unavailable

};

inline std::error_code make_error_code(instrprof_error E) {

  return std::error_code(static_cast<int>(E), instrprof_category());

}

class InstrProfError : public ErrorInfo<InstrProfError> {

public:

  InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine())

      : Err(Err), Msg(ErrStr.str()) {

    assert(Err != instrprof_error::success && "Not an error");

  }

  std::string message() const override;

  void log(raw_ostream &OS) const override { OS << message(); }

  std::error_code convertToErrorCode() const override {

    return make_error_code(Err);

  }

  instrprof_error get() const { return Err; }

  const std::string &getMessage() const { return Msg; }

  /// Consume an Error and return the raw enum value contained within it. The

  /// Error must either be a success value, or contain a single InstrProfError.

  static instrprof_error take(Error E) {

    auto Err = instrprof_error::success;

    handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) {

      assert(Err == instrprof_error::success && "Multiple errors encountered");

      Err = IPE.get();

    });

    return Err;

  }

  static char ID;

private:

  instrprof_error Err;

  std::string Msg;

};

class SoftInstrProfErrors {

  /// Count the number of soft instrprof_errors encountered and keep track of

  /// the first such error for reporting purposes.

  /// The first soft error encountered.

  instrprof_error FirstError = instrprof_error::success;

  /// The number of hash mismatches.

  unsigned NumHashMismatches = 0;

  /// The number of count mismatches.

  unsigned NumCountMismatches = 0;

  /// The number of counter overflows.

  unsigned NumCounterOverflows = 0;

  /// The number of value site count mismatches.

  unsigned NumValueSiteCountMismatches = 0;

public:

  SoftInstrProfErrors() = default;

  ~SoftInstrProfErrors() {

    assert(FirstError == instrprof_error::success &&

           "Unchecked soft error encountered");

  }

  /// Track a soft error (\p IE) and increment its associated counter.

  void addError(instrprof_error IE);

  /// Get the number of hash mismatches.

  unsigned getNumHashMismatches() const { return NumHashMismatches; }

  /// Get the number of count mismatches.

  unsigned getNumCountMismatches() const { return NumCountMismatches; }

  /// Get the number of counter overflows.

  unsigned getNumCounterOverflows() const { return NumCounterOverflows; }

  /// Get the number of value site count mismatches.

  unsigned getNumValueSiteCountMismatches() const {

    return NumValueSiteCountMismatches;

  }

  /// Return the first encountered error and reset FirstError to a success

  /// value.

  Error takeError() {

    if (FirstError == instrprof_error::success)

      return Error::success();

    auto E = make_error<InstrProfError>(FirstError);

    FirstError = instrprof_error::success;

    return E;

  }

};

namespace object {

class SectionRef;

} // end namespace object

namespace IndexedInstrProf {

uint64_t ComputeHash(StringRef K);

} // end namespace IndexedInstrProf

/// A symbol table used for function PGO name look-up with keys

/// (such as pointers, md5hash values) to the function. A function's

/// PGO name or name's md5hash are used in retrieving the profile

/// data of the function. See \c getPGOFuncName() method for details

/// on how PGO name is formed.

class InstrProfSymtab {

public:

  using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;

private:

  StringRef Data;

  uint64_t Address = 0;

  // Unique name strings.

  StringSet<> NameTab;

  // A map from MD5 keys to function name strings.

  std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;

  // A map from MD5 keys to function define. We only populate this map

  // when build the Symtab from a Module.

  std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;

  // A map from function runtime address to function name MD5 hash.

  // This map is only populated and used by raw instr profile reader.

  AddrHashMap AddrToMD5Map;

  bool Sorted = false;

  static StringRef getExternalSymbol() {

    return "** External Symbol **";

  }

  // If the symtab is created by a series of calls to \c addFuncName, \c

  // finalizeSymtab needs to be called before looking up function names.

  // This is required because the underlying map is a vector (for space

  // efficiency) which needs to be sorted.

  inline void finalizeSymtab();

public:

  InstrProfSymtab() = default;

  /// Create InstrProfSymtab from an object file section which

  /// contains function PGO names. When section may contain raw

  /// string data or string data in compressed form. This method

  /// only initialize the symtab with reference to the data and

  /// the section base address. The decompression will be delayed

  /// until before it is used. See also \c create(StringRef) method.

  Error create(object::SectionRef &Section);

  /// This interface is used by reader of CoverageMapping test

  /// format.

  inline Error create(StringRef D, uint64_t BaseAddr);

  /// \c NameStrings is a string composed of one of more sub-strings

  ///  encoded in the format described in \c collectPGOFuncNameStrings.

  /// This method is a wrapper to \c readPGOFuncNameStrings method.

  inline Error create(StringRef NameStrings);

  /// A wrapper interface to populate the PGO symtab with functions

  /// decls from module \c M. This interface is used by transformation

  /// passes such as indirect function call promotion. Variable \c InLTO

  /// indicates if this is called from LTO optimization passes.

  Error create(Module &M, bool InLTO = false);

  /// Create InstrProfSymtab from a set of names iteratable from

  /// \p IterRange. This interface is used by IndexedProfReader.

  template <typename NameIterRange> Error create(const NameIterRange &IterRange);

  /// Update the symtab by adding \p FuncName to the table. This interface

  /// is used by the raw and text profile readers.

  Error addFuncName(StringRef FuncName) {

    if (FuncName.empty())

      return make_error<InstrProfError>(instrprof_error::malformed,

                                        "function name is empty");

    auto Ins = NameTab.insert(FuncName);

    if (Ins.second) {

      MD5NameMap.push_back(std::make_pair(

          IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));

      Sorted = false;

    }

    return Error::success();

  }

  /// Map a function address to its name's MD5 hash. This interface

  /// is only used by the raw profiler reader.

  void mapAddress(uint64_t Addr, uint64_t MD5Val) {

    AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));

  }

  /// Return a function's hash, or 0, if the function isn't in this SymTab.

  uint64_t getFunctionHashFromAddress(uint64_t Address);

  /// Return function's PGO name from the function name's symbol

  /// address in the object file. If an error occurs, return

  /// an empty string.

  StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);

  /// Return function's PGO name from the name's md5 hash value.

  /// If not found, return an empty string.

  inline StringRef getFuncName(uint64_t FuncMD5Hash);

  /// Just like getFuncName, except that it will return a non-empty StringRef

  /// if the function is external to this symbol table. All such cases

  /// will be represented using the same StringRef value.

  inline StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash);

  /// True if Symbol is the value used to represent external symbols.

  static bool isExternalSymbol(const StringRef &Symbol) {

    return Symbol == InstrProfSymtab::getExternalSymbol();

  }

  /// Return function from the name's md5 hash. Return nullptr if not found.

  inline Function *getFunction(uint64_t FuncMD5Hash);

  /// Return the function's original assembly name by stripping off

  /// the prefix attached (to symbols with priviate linkage). For

  /// global functions, it returns the same string as getFuncName.

  inline StringRef getOrigFuncName(uint64_t FuncMD5Hash);

  /// Return the name section data.

  inline StringRef getNameData() const { return Data; }

  /// Dump the symbols in this table.

  void dumpNames(raw_ostream &OS) const {

    for (StringRef S : NameTab.keys())

      OS << S << "\n";

  }

};

Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {

  Data = D;

  Address = BaseAddr;

  return Error::success();

}

Error InstrProfSymtab::create(StringRef NameStrings) {

  return readPGOFuncNameStrings(NameStrings, *this);

}

template <typename NameIterRange>

Error InstrProfSymtab::create(const NameIterRange &IterRange) {

  for (auto Name : IterRange)

    if (Error E = addFuncName(Name))

      return E;

  finalizeSymtab();

  return Error::success();

}

void InstrProfSymtab::finalizeSymtab() {

  if (Sorted)

    return;

  llvm::sort(MD5NameMap, less_first());

  llvm::sort(MD5FuncMap, less_first());

  llvm::sort(AddrToMD5Map, less_first());

  AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),

                     AddrToMD5Map.end());

  Sorted = true;

}

StringRef InstrProfSymtab::getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash) {

  StringRef ret = getFuncName(FuncMD5Hash);

  if (ret.empty())

    return InstrProfSymtab::getExternalSymbol();

  return ret;

}

StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) {

  finalizeSymtab();

  auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash,

                                  [](const std::pair<uint64_t, StringRef> &LHS,

                                     uint64_t RHS) { return LHS.first < RHS; });

  if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)

    return Result->second;

  return StringRef();

}

Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {

  finalizeSymtab();

  auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,

                                  [](const std::pair<uint64_t, Function *> &LHS,

                                     uint64_t RHS) { return LHS.first < RHS; });

  if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)

    return Result->second;

  return nullptr;

}

// See also getPGOFuncName implementation. These two need to be

// matched.

StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) {

  StringRef PGOName = getFuncName(FuncMD5Hash);

  size_t S = PGOName.find_first_of(':');

  if (S == StringRef::npos)

    return PGOName;

  return PGOName.drop_front(S + 1);

}

// To store the sums of profile count values, or the percentage of

// the sums of the total count values.

struct CountSumOrPercent {

  uint64_t NumEntries;

  double CountSum;

  double ValueCounts[IPVK_Last - IPVK_First + 1];

  CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {}

  void reset() {

    NumEntries = 0;

    CountSum = 0.0f;

    for (double &VC : ValueCounts)

      VC = 0.0f;

  }

};

// Function level or program level overlap information.

struct OverlapStats {

  enum OverlapStatsLevel { ProgramLevel, FunctionLevel };

  // Sum of the total count values for the base profile.

  CountSumOrPercent Base;

  // Sum of the total count values for the test profile.

  CountSumOrPercent Test;

  // Overlap lap score. Should be in range of [0.0f to 1.0f].

  CountSumOrPercent Overlap;

  CountSumOrPercent Mismatch;

  CountSumOrPercent Unique;

  OverlapStatsLevel Level;

  const std::string *BaseFilename;

  const std::string *TestFilename;

  StringRef FuncName;

  uint64_t FuncHash;

  bool Valid;

  OverlapStats(OverlapStatsLevel L = ProgramLevel)

      : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0),

        Valid(false) {}

  void dump(raw_fd_ostream &OS) const;

  void setFuncInfo(StringRef Name, uint64_t Hash) {

    FuncName = Name;

    FuncHash = Hash;

  }

  Error accumulateCounts(const std::string &BaseFilename,

                         const std::string &TestFilename, bool IsCS);

  void addOneMismatch(const CountSumOrPercent &MismatchFunc);

  void addOneUnique(const CountSumOrPercent &UniqueFunc);

  static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,

                             double Sum2) {

    if (Sum1 < 1.0f || Sum2 < 1.0f)

      return 0.0f;

    return std::min(Val1 / Sum1, Val2 / Sum2);

  }

};

// This is used to filter the functions whose overlap information

// to be output.

struct OverlapFuncFilters {

  uint64_t ValueCutoff;

  const std::string NameFilter;

};

struct InstrProfValueSiteRecord {

  /// Value profiling data pairs at a given value site.

  std::list<InstrProfValueData> ValueData;

  InstrProfValueSiteRecord() { ValueData.clear(); }

  template <class InputIterator>

  InstrProfValueSiteRecord(InputIterator F, InputIterator L)

      : ValueData(F, L) {}

  /// Sort ValueData ascending by Value

  void sortByTargetValues() {

    ValueData.sort(

        [](const InstrProfValueData &left, const InstrProfValueData &right) {

          return left.Value < right.Value;

        });

  }

  /// Sort ValueData Descending by Count

  inline void sortByCount();

  /// Merge data from another InstrProfValueSiteRecord

  /// Optionally scale merged counts by \p Weight.

  void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,

             function_ref<void(instrprof_error)> Warn);

  /// Scale up value profile data counts by N (Numerator) / D (Denominator).

  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);

  /// Compute the overlap b/w this record and Input record.

  void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,

               OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);

};

/// Profiling information for a single function.

struct InstrProfRecord {

  std::vector<uint64_t> Counts;

  InstrProfRecord() = default;

  InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}

  InstrProfRecord(InstrProfRecord &&) = default;

  InstrProfRecord(const InstrProfRecord &RHS)

      : Counts(RHS.Counts),

        ValueData(RHS.ValueData

                      ? std::make_unique<ValueProfData>(*RHS.ValueData)

                      : nullptr) {}

  InstrProfRecord &operator=(InstrProfRecord &&) = default;

  InstrProfRecord &operator=(const InstrProfRecord &RHS) {

    Counts = RHS.Counts;

    if (!RHS.ValueData) {

      ValueData = nullptr;

      return *this;

    }

    if (!ValueData)

      ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);

    else

      *ValueData = *RHS.ValueData;

    return *this;

  }

  /// Return the number of value profile kinds with non-zero number

  /// of profile sites.

  inline uint32_t getNumValueKinds() const;

  /// Return the number of instrumented sites for ValueKind.

  inline uint32_t getNumValueSites(uint32_t ValueKind) const;

  /// Return the total number of ValueData for ValueKind.

  inline uint32_t getNumValueData(uint32_t ValueKind) const;

  /// Return the number of value data collected for ValueKind at profiling

  /// site: Site.

  inline uint32_t getNumValueDataForSite(uint32_t ValueKind,

                                         uint32_t Site) const;

  /// Return the array of profiled values at \p Site. If \p TotalC

  /// is not null, the total count of all target values at this site

  /// will be stored in \c *TotalC.

  inline std::unique_ptr<InstrProfValueData[]>

  getValueForSite(uint32_t ValueKind, uint32_t Site,

                  uint64_t *TotalC = nullptr) const;

  /// Get the target value/counts of kind \p ValueKind collected at site

  /// \p Site and store the result in array \p Dest. Return the total

  /// counts of all target values at this site.

  inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,

                                  uint32_t Site) const;

  /// Reserve space for NumValueSites sites.

  inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);

  /// Add ValueData for ValueKind at value Site.

  void addValueData(uint32_t ValueKind, uint32_t Site,

                    InstrProfValueData *VData, uint32_t N,

                    InstrProfSymtab *SymTab);

  /// Merge the counts in \p Other into this one.

  /// Optionally scale merged counts by \p Weight.

  void merge(InstrProfRecord &Other, uint64_t Weight,

             function_ref<void(instrprof_error)> Warn);

  /// Scale up profile counts (including value profile data) by

  /// a factor of (N / D).

  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);

  /// Sort value profile data (per site) by count.

  void sortValueData() {

    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

      for (auto &SR : getValueSitesForKind(Kind))

        SR.sortByCount();

  }

  /// Clear value data entries and edge counters.

  void Clear() {

    Counts.clear();

    clearValueData();

  }

  /// Clear value data entries

  void clearValueData() { ValueData = nullptr; }

  /// Compute the sums of all counts and store in Sum.

  void accumulateCounts(CountSumOrPercent &Sum) const;

  /// Compute the overlap b/w this IntrprofRecord and Other.

  void overlap(InstrProfRecord &Other, OverlapStats &Overlap,

               OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);

  /// Compute the overlap of value profile counts.

  void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,

                            OverlapStats &Overlap,

                            OverlapStats &FuncLevelOverlap);

  enum CountPseudoKind {

    NotPseudo = 0,

    PseudoHot,

    PseudoWarm,

  };

  enum PseudoCountVal {

    HotFunctionVal = -1,

    WarmFunctionVal = -2,

  };

  CountPseudoKind getCountPseudoKind() const {

    uint64_t FirstCount = Counts[0];

    if (FirstCount == (uint64_t)HotFunctionVal)