Details | Last modification | View Log | RSS feed
| Rev | Author | Line No. | Line |
|---|---|---|---|
| 14 | pmbaty | 1 | //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===// |
| 2 | // |
||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
||
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
||
| 6 | // |
||
| 7 | //===----------------------------------------------------------------------===// |
||
| 8 | // |
||
| 9 | // This file contains support for reading profiling data for instrumentation |
||
| 10 | // based PGO and coverage. |
||
| 11 | // |
||
| 12 | //===----------------------------------------------------------------------===// |
||
| 13 | |||
| 14 | #ifndef LLVM_PROFILEDATA_INSTRPROFREADER_H |
||
| 15 | #define LLVM_PROFILEDATA_INSTRPROFREADER_H |
||
| 16 | |||
| 17 | #include "llvm/ADT/ArrayRef.h" |
||
| 18 | #include "llvm/ADT/StringRef.h" |
||
| 19 | #include "llvm/IR/ProfileSummary.h" |
||
| 20 | #include "llvm/Object/BuildID.h" |
||
| 21 | #include "llvm/ProfileData/InstrProf.h" |
||
| 22 | #include "llvm/ProfileData/InstrProfCorrelator.h" |
||
| 23 | #include "llvm/ProfileData/MemProf.h" |
||
| 24 | #include "llvm/Support/Endian.h" |
||
| 25 | #include "llvm/Support/Error.h" |
||
| 26 | #include "llvm/Support/LineIterator.h" |
||
| 27 | #include "llvm/Support/MathExtras.h" |
||
| 28 | #include "llvm/Support/MemoryBuffer.h" |
||
| 29 | #include "llvm/Support/OnDiskHashTable.h" |
||
| 30 | #include "llvm/Support/SwapByteOrder.h" |
||
| 31 | #include <algorithm> |
||
| 32 | #include <cassert> |
||
| 33 | #include <cstddef> |
||
| 34 | #include <cstdint> |
||
| 35 | #include <iterator> |
||
| 36 | #include <memory> |
||
| 37 | #include <utility> |
||
| 38 | #include <vector> |
||
| 39 | |||
| 40 | namespace llvm { |
||
| 41 | |||
| 42 | class InstrProfReader; |
||
| 43 | |||
| 44 | /// A file format agnostic iterator over profiling data. |
||
| 45 | template <class record_type = NamedInstrProfRecord, |
||
| 46 | class reader_type = InstrProfReader> |
||
| 47 | class InstrProfIterator { |
||
| 48 | public: |
||
| 49 | using iterator_category = std::input_iterator_tag; |
||
| 50 | using value_type = record_type; |
||
| 51 | using difference_type = std::ptrdiff_t; |
||
| 52 | using pointer = value_type *; |
||
| 53 | using reference = value_type &; |
||
| 54 | |||
| 55 | private: |
||
| 56 | reader_type *Reader = nullptr; |
||
| 57 | value_type Record; |
||
| 58 | |||
| 59 | void increment() { |
||
| 60 | if (Error E = Reader->readNextRecord(Record)) { |
||
| 61 | // Handle errors in the reader. |
||
| 62 | InstrProfError::take(std::move(E)); |
||
| 63 | *this = InstrProfIterator(); |
||
| 64 | } |
||
| 65 | } |
||
| 66 | |||
| 67 | public: |
||
| 68 | InstrProfIterator() = default; |
||
| 69 | InstrProfIterator(reader_type *Reader) : Reader(Reader) { increment(); } |
||
| 70 | |||
| 71 | InstrProfIterator &operator++() { |
||
| 72 | increment(); |
||
| 73 | return *this; |
||
| 74 | } |
||
| 75 | bool operator==(const InstrProfIterator &RHS) const { |
||
| 76 | return Reader == RHS.Reader; |
||
| 77 | } |
||
| 78 | bool operator!=(const InstrProfIterator &RHS) const { |
||
| 79 | return Reader != RHS.Reader; |
||
| 80 | } |
||
| 81 | value_type &operator*() { return Record; } |
||
| 82 | value_type *operator->() { return &Record; } |
||
| 83 | }; |
||
| 84 | |||
| 85 | /// Base class and interface for reading profiling data of any known instrprof |
||
| 86 | /// format. Provides an iterator over NamedInstrProfRecords. |
||
| 87 | class InstrProfReader { |
||
| 88 | instrprof_error LastError = instrprof_error::success; |
||
| 89 | std::string LastErrorMsg; |
||
| 90 | |||
| 91 | public: |
||
| 92 | InstrProfReader() = default; |
||
| 93 | virtual ~InstrProfReader() = default; |
||
| 94 | |||
| 95 | /// Read the header. Required before reading first record. |
||
| 96 | virtual Error readHeader() = 0; |
||
| 97 | |||
| 98 | /// Read a single record. |
||
| 99 | virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0; |
||
| 100 | |||
| 101 | /// Read a list of binary ids. |
||
| 102 | virtual Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) { |
||
| 103 | return success(); |
||
| 104 | } |
||
| 105 | |||
| 106 | /// Print binary ids. |
||
| 107 | virtual Error printBinaryIds(raw_ostream &OS) { return success(); }; |
||
| 108 | |||
| 109 | /// Iterator over profile data. |
||
| 110 | InstrProfIterator<> begin() { return InstrProfIterator<>(this); } |
||
| 111 | InstrProfIterator<> end() { return InstrProfIterator<>(); } |
||
| 112 | |||
| 113 | /// Return the profile version. |
||
| 114 | virtual uint64_t getVersion() const = 0; |
||
| 115 | |||
| 116 | virtual bool isIRLevelProfile() const = 0; |
||
| 117 | |||
| 118 | virtual bool hasCSIRLevelProfile() const = 0; |
||
| 119 | |||
| 120 | virtual bool instrEntryBBEnabled() const = 0; |
||
| 121 | |||
| 122 | /// Return true if we must provide debug info to create PGO profiles. |
||
| 123 | virtual bool useDebugInfoCorrelate() const { return false; } |
||
| 124 | |||
| 125 | /// Return true if the profile has single byte counters representing coverage. |
||
| 126 | virtual bool hasSingleByteCoverage() const = 0; |
||
| 127 | |||
| 128 | /// Return true if the profile only instruments function entries. |
||
| 129 | virtual bool functionEntryOnly() const = 0; |
||
| 130 | |||
| 131 | /// Return true if profile includes a memory profile. |
||
| 132 | virtual bool hasMemoryProfile() const = 0; |
||
| 133 | |||
| 134 | /// Returns a BitsetEnum describing the attributes of the profile. To check |
||
| 135 | /// individual attributes prefer using the helpers above. |
||
| 136 | virtual InstrProfKind getProfileKind() const = 0; |
||
| 137 | |||
| 138 | /// Return the PGO symtab. There are three different readers: |
||
| 139 | /// Raw, Text, and Indexed profile readers. The first two types |
||
| 140 | /// of readers are used only by llvm-profdata tool, while the indexed |
||
| 141 | /// profile reader is also used by llvm-cov tool and the compiler ( |
||
| 142 | /// backend or frontend). Since creating PGO symtab can create |
||
| 143 | /// significant runtime and memory overhead (as it touches data |
||
| 144 | /// for the whole program), InstrProfSymtab for the indexed profile |
||
| 145 | /// reader should be created on demand and it is recommended to be |
||
| 146 | /// only used for dumping purpose with llvm-proftool, not with the |
||
| 147 | /// compiler. |
||
| 148 | virtual InstrProfSymtab &getSymtab() = 0; |
||
| 149 | |||
| 150 | /// Compute the sum of counts and return in Sum. |
||
| 151 | void accumulateCounts(CountSumOrPercent &Sum, bool IsCS); |
||
| 152 | |||
| 153 | protected: |
||
| 154 | std::unique_ptr<InstrProfSymtab> Symtab; |
||
| 155 | |||
| 156 | /// Set the current error and return same. |
||
| 157 | Error error(instrprof_error Err, const std::string &ErrMsg = "") { |
||
| 158 | LastError = Err; |
||
| 159 | LastErrorMsg = ErrMsg; |
||
| 160 | if (Err == instrprof_error::success) |
||
| 161 | return Error::success(); |
||
| 162 | return make_error<InstrProfError>(Err, ErrMsg); |
||
| 163 | } |
||
| 164 | |||
| 165 | Error error(Error &&E) { |
||
| 166 | handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { |
||
| 167 | LastError = IPE.get(); |
||
| 168 | LastErrorMsg = IPE.getMessage(); |
||
| 169 | }); |
||
| 170 | return make_error<InstrProfError>(LastError, LastErrorMsg); |
||
| 171 | } |
||
| 172 | |||
| 173 | /// Clear the current error and return a successful one. |
||
| 174 | Error success() { return error(instrprof_error::success); } |
||
| 175 | |||
| 176 | public: |
||
| 177 | /// Return true if the reader has finished reading the profile data. |
||
| 178 | bool isEOF() { return LastError == instrprof_error::eof; } |
||
| 179 | |||
| 180 | /// Return true if the reader encountered an error reading profiling data. |
||
| 181 | bool hasError() { return LastError != instrprof_error::success && !isEOF(); } |
||
| 182 | |||
| 183 | /// Get the current error. |
||
| 184 | Error getError() { |
||
| 185 | if (hasError()) |
||
| 186 | return make_error<InstrProfError>(LastError, LastErrorMsg); |
||
| 187 | return Error::success(); |
||
| 188 | } |
||
| 189 | |||
| 190 | /// Factory method to create an appropriately typed reader for the given |
||
| 191 | /// instrprof file. |
||
| 192 | static Expected<std::unique_ptr<InstrProfReader>> |
||
| 193 | create(const Twine &Path, const InstrProfCorrelator *Correlator = nullptr); |
||
| 194 | |||
| 195 | static Expected<std::unique_ptr<InstrProfReader>> |
||
| 196 | create(std::unique_ptr<MemoryBuffer> Buffer, |
||
| 197 | const InstrProfCorrelator *Correlator = nullptr); |
||
| 198 | }; |
||
| 199 | |||
| 200 | /// Reader for the simple text based instrprof format. |
||
| 201 | /// |
||
| 202 | /// This format is a simple text format that's suitable for test data. Records |
||
| 203 | /// are separated by one or more blank lines, and record fields are separated by |
||
| 204 | /// new lines. |
||
| 205 | /// |
||
| 206 | /// Each record consists of a function name, a function hash, a number of |
||
| 207 | /// counters, and then each counter value, in that order. |
||
| 208 | class TextInstrProfReader : public InstrProfReader { |
||
| 209 | private: |
||
| 210 | /// The profile data file contents. |
||
| 211 | std::unique_ptr<MemoryBuffer> DataBuffer; |
||
| 212 | /// Iterator over the profile data. |
||
| 213 | line_iterator Line; |
||
| 214 | /// The attributes of the current profile. |
||
| 215 | InstrProfKind ProfileKind = InstrProfKind::Unknown; |
||
| 216 | |||
| 217 | Error readValueProfileData(InstrProfRecord &Record); |
||
| 218 | |||
| 219 | public: |
||
| 220 | TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_) |
||
| 221 | : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} |
||
| 222 | TextInstrProfReader(const TextInstrProfReader &) = delete; |
||
| 223 | TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; |
||
| 224 | |||
| 225 | /// Return true if the given buffer is in text instrprof format. |
||
| 226 | static bool hasFormat(const MemoryBuffer &Buffer); |
||
| 227 | |||
| 228 | // Text format does not have version, so return 0. |
||
| 229 | uint64_t getVersion() const override { return 0; } |
||
| 230 | |||
| 231 | bool isIRLevelProfile() const override { |
||
| 232 | return static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation); |
||
| 233 | } |
||
| 234 | |||
| 235 | bool hasCSIRLevelProfile() const override { |
||
| 236 | return static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive); |
||
| 237 | } |
||
| 238 | |||
| 239 | bool instrEntryBBEnabled() const override { |
||
| 240 | return static_cast<bool>(ProfileKind & |
||
| 241 | InstrProfKind::FunctionEntryInstrumentation); |
||
| 242 | } |
||
| 243 | |||
| 244 | bool hasSingleByteCoverage() const override { |
||
| 245 | return static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage); |
||
| 246 | } |
||
| 247 | |||
| 248 | bool functionEntryOnly() const override { |
||
| 249 | return static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly); |
||
| 250 | } |
||
| 251 | |||
| 252 | bool hasMemoryProfile() const override { |
||
| 253 | // TODO: Add support for text format memory profiles. |
||
| 254 | return false; |
||
| 255 | } |
||
| 256 | |||
| 257 | InstrProfKind getProfileKind() const override { return ProfileKind; } |
||
| 258 | |||
| 259 | /// Read the header. |
||
| 260 | Error readHeader() override; |
||
| 261 | |||
| 262 | /// Read a single record. |
||
| 263 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
||
| 264 | |||
| 265 | InstrProfSymtab &getSymtab() override { |
||
| 266 | assert(Symtab); |
||
| 267 | return *Symtab; |
||
| 268 | } |
||
| 269 | }; |
||
| 270 | |||
| 271 | /// Reader for the raw instrprof binary format from runtime. |
||
| 272 | /// |
||
| 273 | /// This format is a raw memory dump of the instrumentation-based profiling data |
||
| 274 | /// from the runtime. It has no index. |
||
| 275 | /// |
||
| 276 | /// Templated on the unsigned type whose size matches pointers on the platform |
||
| 277 | /// that wrote the profile. |
||
| 278 | template <class IntPtrT> |
||
| 279 | class RawInstrProfReader : public InstrProfReader { |
||
| 280 | private: |
||
| 281 | /// The profile data file contents. |
||
| 282 | std::unique_ptr<MemoryBuffer> DataBuffer; |
||
| 283 | /// If available, this hold the ProfileData array used to correlate raw |
||
| 284 | /// instrumentation data to their functions. |
||
| 285 | const InstrProfCorrelatorImpl<IntPtrT> *Correlator; |
||
| 286 | bool ShouldSwapBytes; |
||
| 287 | // The value of the version field of the raw profile data header. The lower 56 |
||
| 288 | // bits specifies the format version and the most significant 8 bits specify |
||
| 289 | // the variant types of the profile. |
||
| 290 | uint64_t Version; |
||
| 291 | uint64_t CountersDelta; |
||
| 292 | uint64_t NamesDelta; |
||
| 293 | const RawInstrProf::ProfileData<IntPtrT> *Data; |
||
| 294 | const RawInstrProf::ProfileData<IntPtrT> *DataEnd; |
||
| 295 | const char *CountersStart; |
||
| 296 | const char *CountersEnd; |
||
| 297 | const char *NamesStart; |
||
| 298 | const char *NamesEnd; |
||
| 299 | // After value profile is all read, this pointer points to |
||
| 300 | // the header of next profile data (if exists) |
||
| 301 | const uint8_t *ValueDataStart; |
||
| 302 | uint32_t ValueKindLast; |
||
| 303 | uint32_t CurValueDataSize; |
||
| 304 | |||
| 305 | /// Total size of binary ids. |
||
| 306 | uint64_t BinaryIdsSize{0}; |
||
| 307 | /// Start address of binary id length and data pairs. |
||
| 308 | const uint8_t *BinaryIdsStart; |
||
| 309 | |||
| 310 | public: |
||
| 311 | RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer, |
||
| 312 | const InstrProfCorrelator *Correlator) |
||
| 313 | : DataBuffer(std::move(DataBuffer)), |
||
| 314 | Correlator(dyn_cast_or_null<const InstrProfCorrelatorImpl<IntPtrT>>( |
||
| 315 | Correlator)) {} |
||
| 316 | RawInstrProfReader(const RawInstrProfReader &) = delete; |
||
| 317 | RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; |
||
| 318 | |||
| 319 | static bool hasFormat(const MemoryBuffer &DataBuffer); |
||
| 320 | Error readHeader() override; |
||
| 321 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
||
| 322 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; |
||
| 323 | Error printBinaryIds(raw_ostream &OS) override; |
||
| 324 | |||
| 325 | uint64_t getVersion() const override { return Version; } |
||
| 326 | |||
| 327 | bool isIRLevelProfile() const override { |
||
| 328 | return (Version & VARIANT_MASK_IR_PROF) != 0; |
||
| 329 | } |
||
| 330 | |||
| 331 | bool hasCSIRLevelProfile() const override { |
||
| 332 | return (Version & VARIANT_MASK_CSIR_PROF) != 0; |
||
| 333 | } |
||
| 334 | |||
| 335 | bool instrEntryBBEnabled() const override { |
||
| 336 | return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; |
||
| 337 | } |
||
| 338 | |||
| 339 | bool useDebugInfoCorrelate() const override { |
||
| 340 | return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; |
||
| 341 | } |
||
| 342 | |||
| 343 | bool hasSingleByteCoverage() const override { |
||
| 344 | return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; |
||
| 345 | } |
||
| 346 | |||
| 347 | bool functionEntryOnly() const override { |
||
| 348 | return (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; |
||
| 349 | } |
||
| 350 | |||
| 351 | bool hasMemoryProfile() const override { |
||
| 352 | // Memory profiles have a separate raw format, so this should never be set. |
||
| 353 | assert(!(Version & VARIANT_MASK_MEMPROF)); |
||
| 354 | return false; |
||
| 355 | } |
||
| 356 | |||
| 357 | /// Returns a BitsetEnum describing the attributes of the raw instr profile. |
||
| 358 | InstrProfKind getProfileKind() const override; |
||
| 359 | |||
| 360 | InstrProfSymtab &getSymtab() override { |
||
| 361 | assert(Symtab.get()); |
||
| 362 | return *Symtab.get(); |
||
| 363 | } |
||
| 364 | |||
| 365 | private: |
||
| 366 | Error createSymtab(InstrProfSymtab &Symtab); |
||
| 367 | Error readNextHeader(const char *CurrentPos); |
||
| 368 | Error readHeader(const RawInstrProf::Header &Header); |
||
| 369 | |||
| 370 | template <class IntT> IntT swap(IntT Int) const { |
||
| 371 | return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int; |
||
| 372 | } |
||
| 373 | |||
| 374 | support::endianness getDataEndianness() const { |
||
| 375 | support::endianness HostEndian = getHostEndianness(); |
||
| 376 | if (!ShouldSwapBytes) |
||
| 377 | return HostEndian; |
||
| 378 | if (HostEndian == support::little) |
||
| 379 | return support::big; |
||
| 380 | else |
||
| 381 | return support::little; |
||
| 382 | } |
||
| 383 | |||
| 384 | inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) { |
||
| 385 | return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); |
||
| 386 | } |
||
| 387 | |||
| 388 | Error readName(NamedInstrProfRecord &Record); |
||
| 389 | Error readFuncHash(NamedInstrProfRecord &Record); |
||
| 390 | Error readRawCounts(InstrProfRecord &Record); |
||
| 391 | Error readValueProfilingData(InstrProfRecord &Record); |
||
| 392 | bool atEnd() const { return Data == DataEnd; } |
||
| 393 | |||
| 394 | void advanceData() { |
||
| 395 | // `CountersDelta` is a constant zero when using debug info correlation. |
||
| 396 | if (!Correlator) { |
||
| 397 | // The initial CountersDelta is the in-memory address difference between |
||
| 398 | // the data and counts sections: |
||
| 399 | // start(__llvm_prf_cnts) - start(__llvm_prf_data) |
||
| 400 | // As we advance to the next record, we maintain the correct CountersDelta |
||
| 401 | // with respect to the next record. |
||
| 402 | CountersDelta -= sizeof(*Data); |
||
| 403 | } |
||
| 404 | Data++; |
||
| 405 | ValueDataStart += CurValueDataSize; |
||
| 406 | } |
||
| 407 | |||
| 408 | const char *getNextHeaderPos() const { |
||
| 409 | assert(atEnd()); |
||
| 410 | return (const char *)ValueDataStart; |
||
| 411 | } |
||
| 412 | |||
| 413 | StringRef getName(uint64_t NameRef) const { |
||
| 414 | return Symtab->getFuncName(swap(NameRef)); |
||
| 415 | } |
||
| 416 | |||
| 417 | int getCounterTypeSize() const { |
||
| 418 | return hasSingleByteCoverage() ? sizeof(uint8_t) : sizeof(uint64_t); |
||
| 419 | } |
||
| 420 | }; |
||
| 421 | |||
| 422 | using RawInstrProfReader32 = RawInstrProfReader<uint32_t>; |
||
| 423 | using RawInstrProfReader64 = RawInstrProfReader<uint64_t>; |
||
| 424 | |||
| 425 | namespace IndexedInstrProf { |
||
| 426 | |||
| 427 | enum class HashT : uint32_t; |
||
| 428 | |||
| 429 | } // end namespace IndexedInstrProf |
||
| 430 | |||
| 431 | /// Trait for lookups into the on-disk hash table for the binary instrprof |
||
| 432 | /// format. |
||
| 433 | class InstrProfLookupTrait { |
||
| 434 | std::vector<NamedInstrProfRecord> DataBuffer; |
||
| 435 | IndexedInstrProf::HashT HashType; |
||
| 436 | unsigned FormatVersion; |
||
| 437 | // Endianness of the input value profile data. |
||
| 438 | // It should be LE by default, but can be changed |
||
| 439 | // for testing purpose. |
||
| 440 | support::endianness ValueProfDataEndianness = support::little; |
||
| 441 | |||
| 442 | public: |
||
| 443 | InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion) |
||
| 444 | : HashType(HashType), FormatVersion(FormatVersion) {} |
||
| 445 | |||
| 446 | using data_type = ArrayRef<NamedInstrProfRecord>; |
||
| 447 | |||
| 448 | using internal_key_type = StringRef; |
||
| 449 | using external_key_type = StringRef; |
||
| 450 | using hash_value_type = uint64_t; |
||
| 451 | using offset_type = uint64_t; |
||
| 452 | |||
| 453 | static bool EqualKey(StringRef A, StringRef B) { return A == B; } |
||
| 454 | static StringRef GetInternalKey(StringRef K) { return K; } |
||
| 455 | static StringRef GetExternalKey(StringRef K) { return K; } |
||
| 456 | |||
| 457 | hash_value_type ComputeHash(StringRef K); |
||
| 458 | |||
| 459 | static std::pair<offset_type, offset_type> |
||
| 460 | ReadKeyDataLength(const unsigned char *&D) { |
||
| 461 | using namespace support; |
||
| 462 | |||
| 463 | offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D); |
||
| 464 | offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D); |
||
| 465 | return std::make_pair(KeyLen, DataLen); |
||
| 466 | } |
||
| 467 | |||
| 468 | StringRef ReadKey(const unsigned char *D, offset_type N) { |
||
| 469 | return StringRef((const char *)D, N); |
||
| 470 | } |
||
| 471 | |||
| 472 | bool readValueProfilingData(const unsigned char *&D, |
||
| 473 | const unsigned char *const End); |
||
| 474 | data_type ReadData(StringRef K, const unsigned char *D, offset_type N); |
||
| 475 | |||
| 476 | // Used for testing purpose only. |
||
| 477 | void setValueProfDataEndianness(support::endianness Endianness) { |
||
| 478 | ValueProfDataEndianness = Endianness; |
||
| 479 | } |
||
| 480 | }; |
||
| 481 | |||
| 482 | struct InstrProfReaderIndexBase { |
||
| 483 | virtual ~InstrProfReaderIndexBase() = default; |
||
| 484 | |||
| 485 | // Read all the profile records with the same key pointed to the current |
||
| 486 | // iterator. |
||
| 487 | virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0; |
||
| 488 | |||
| 489 | // Read all the profile records with the key equal to FuncName |
||
| 490 | virtual Error getRecords(StringRef FuncName, |
||
| 491 | ArrayRef<NamedInstrProfRecord> &Data) = 0; |
||
| 492 | virtual void advanceToNextKey() = 0; |
||
| 493 | virtual bool atEnd() const = 0; |
||
| 494 | virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; |
||
| 495 | virtual uint64_t getVersion() const = 0; |
||
| 496 | virtual bool isIRLevelProfile() const = 0; |
||
| 497 | virtual bool hasCSIRLevelProfile() const = 0; |
||
| 498 | virtual bool instrEntryBBEnabled() const = 0; |
||
| 499 | virtual bool hasSingleByteCoverage() const = 0; |
||
| 500 | virtual bool functionEntryOnly() const = 0; |
||
| 501 | virtual bool hasMemoryProfile() const = 0; |
||
| 502 | virtual InstrProfKind getProfileKind() const = 0; |
||
| 503 | virtual Error populateSymtab(InstrProfSymtab &) = 0; |
||
| 504 | }; |
||
| 505 | |||
| 506 | using OnDiskHashTableImplV3 = |
||
| 507 | OnDiskIterableChainedHashTable<InstrProfLookupTrait>; |
||
| 508 | |||
| 509 | using MemProfRecordHashTable = |
||
| 510 | OnDiskIterableChainedHashTable<memprof::RecordLookupTrait>; |
||
| 511 | using MemProfFrameHashTable = |
||
| 512 | OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>; |
||
| 513 | |||
| 514 | template <typename HashTableImpl> |
||
| 515 | class InstrProfReaderItaniumRemapper; |
||
| 516 | |||
| 517 | template <typename HashTableImpl> |
||
| 518 | class InstrProfReaderIndex : public InstrProfReaderIndexBase { |
||
| 519 | private: |
||
| 520 | std::unique_ptr<HashTableImpl> HashTable; |
||
| 521 | typename HashTableImpl::data_iterator RecordIterator; |
||
| 522 | uint64_t FormatVersion; |
||
| 523 | |||
| 524 | friend class InstrProfReaderItaniumRemapper<HashTableImpl>; |
||
| 525 | |||
| 526 | public: |
||
| 527 | InstrProfReaderIndex(const unsigned char *Buckets, |
||
| 528 | const unsigned char *const Payload, |
||
| 529 | const unsigned char *const Base, |
||
| 530 | IndexedInstrProf::HashT HashType, uint64_t Version); |
||
| 531 | ~InstrProfReaderIndex() override = default; |
||
| 532 | |||
| 533 | Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override; |
||
| 534 | Error getRecords(StringRef FuncName, |
||
| 535 | ArrayRef<NamedInstrProfRecord> &Data) override; |
||
| 536 | void advanceToNextKey() override { RecordIterator++; } |
||
| 537 | |||
| 538 | bool atEnd() const override { |
||
| 539 | return RecordIterator == HashTable->data_end(); |
||
| 540 | } |
||
| 541 | |||
| 542 | void setValueProfDataEndianness(support::endianness Endianness) override { |
||
| 543 | HashTable->getInfoObj().setValueProfDataEndianness(Endianness); |
||
| 544 | } |
||
| 545 | |||
| 546 | uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } |
||
| 547 | |||
| 548 | bool isIRLevelProfile() const override { |
||
| 549 | return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; |
||
| 550 | } |
||
| 551 | |||
| 552 | bool hasCSIRLevelProfile() const override { |
||
| 553 | return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; |
||
| 554 | } |
||
| 555 | |||
| 556 | bool instrEntryBBEnabled() const override { |
||
| 557 | return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0; |
||
| 558 | } |
||
| 559 | |||
| 560 | bool hasSingleByteCoverage() const override { |
||
| 561 | return (FormatVersion & VARIANT_MASK_BYTE_COVERAGE) != 0; |
||
| 562 | } |
||
| 563 | |||
| 564 | bool functionEntryOnly() const override { |
||
| 565 | return (FormatVersion & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; |
||
| 566 | } |
||
| 567 | |||
| 568 | bool hasMemoryProfile() const override { |
||
| 569 | return (FormatVersion & VARIANT_MASK_MEMPROF) != 0; |
||
| 570 | } |
||
| 571 | |||
| 572 | InstrProfKind getProfileKind() const override; |
||
| 573 | |||
| 574 | Error populateSymtab(InstrProfSymtab &Symtab) override { |
||
| 575 | return Symtab.create(HashTable->keys()); |
||
| 576 | } |
||
| 577 | }; |
||
| 578 | |||
| 579 | /// Name matcher supporting fuzzy matching of symbol names to names in profiles. |
||
| 580 | class InstrProfReaderRemapper { |
||
| 581 | public: |
||
| 582 | virtual ~InstrProfReaderRemapper() = default; |
||
| 583 | virtual Error populateRemappings() { return Error::success(); } |
||
| 584 | virtual Error getRecords(StringRef FuncName, |
||
| 585 | ArrayRef<NamedInstrProfRecord> &Data) = 0; |
||
| 586 | }; |
||
| 587 | |||
| 588 | /// Reader for the indexed binary instrprof format. |
||
| 589 | class IndexedInstrProfReader : public InstrProfReader { |
||
| 590 | private: |
||
| 591 | /// The profile data file contents. |
||
| 592 | std::unique_ptr<MemoryBuffer> DataBuffer; |
||
| 593 | /// The profile remapping file contents. |
||
| 594 | std::unique_ptr<MemoryBuffer> RemappingBuffer; |
||
| 595 | /// The index into the profile data. |
||
| 596 | std::unique_ptr<InstrProfReaderIndexBase> Index; |
||
| 597 | /// The profile remapping file contents. |
||
| 598 | std::unique_ptr<InstrProfReaderRemapper> Remapper; |
||
| 599 | /// Profile summary data. |
||
| 600 | std::unique_ptr<ProfileSummary> Summary; |
||
| 601 | /// Context sensitive profile summary data. |
||
| 602 | std::unique_ptr<ProfileSummary> CS_Summary; |
||
| 603 | /// MemProf profile schema (if available). |
||
| 604 | memprof::MemProfSchema Schema; |
||
| 605 | /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName). |
||
| 606 | std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable; |
||
| 607 | /// MemProf frame profile data on-disk indexed via frame id. |
||
| 608 | std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable; |
||
| 609 | /// Total size of binary ids. |
||
| 610 | uint64_t BinaryIdsSize{0}; |
||
| 611 | /// Start address of binary id length and data pairs. |
||
| 612 | const uint8_t *BinaryIdsStart = nullptr; |
||
| 613 | |||
| 614 | // Index to the current record in the record array. |
||
| 615 | unsigned RecordIndex; |
||
| 616 | |||
| 617 | // Read the profile summary. Return a pointer pointing to one byte past the |
||
| 618 | // end of the summary data if it exists or the input \c Cur. |
||
| 619 | // \c UseCS indicates whether to use the context-sensitive profile summary. |
||
| 620 | const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, |
||
| 621 | const unsigned char *Cur, bool UseCS); |
||
| 622 | |||
| 623 | public: |
||
| 624 | IndexedInstrProfReader( |
||
| 625 | std::unique_ptr<MemoryBuffer> DataBuffer, |
||
| 626 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr) |
||
| 627 | : DataBuffer(std::move(DataBuffer)), |
||
| 628 | RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {} |
||
| 629 | IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; |
||
| 630 | IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; |
||
| 631 | |||
| 632 | /// Return the profile version. |
||
| 633 | uint64_t getVersion() const override { return Index->getVersion(); } |
||
| 634 | bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } |
||
| 635 | bool hasCSIRLevelProfile() const override { |
||
| 636 | return Index->hasCSIRLevelProfile(); |
||
| 637 | } |
||
| 638 | |||
| 639 | bool instrEntryBBEnabled() const override { |
||
| 640 | return Index->instrEntryBBEnabled(); |
||
| 641 | } |
||
| 642 | |||
| 643 | bool hasSingleByteCoverage() const override { |
||
| 644 | return Index->hasSingleByteCoverage(); |
||
| 645 | } |
||
| 646 | |||
| 647 | bool functionEntryOnly() const override { return Index->functionEntryOnly(); } |
||
| 648 | |||
| 649 | bool hasMemoryProfile() const override { return Index->hasMemoryProfile(); } |
||
| 650 | |||
| 651 | /// Returns a BitsetEnum describing the attributes of the indexed instr |
||
| 652 | /// profile. |
||
| 653 | InstrProfKind getProfileKind() const override { |
||
| 654 | return Index->getProfileKind(); |
||
| 655 | } |
||
| 656 | |||
| 657 | /// Return true if the given buffer is in an indexed instrprof format. |
||
| 658 | static bool hasFormat(const MemoryBuffer &DataBuffer); |
||
| 659 | |||
| 660 | /// Read the file header. |
||
| 661 | Error readHeader() override; |
||
| 662 | /// Read a single record. |
||
| 663 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
||
| 664 | |||
| 665 | /// Return the NamedInstrProfRecord associated with FuncName and FuncHash. |
||
| 666 | /// When return a hash_mismatch error and MismatchedFuncSum is not nullptr, |
||
| 667 | /// the sum of all counters in the mismatched function will be set to |
||
| 668 | /// MismatchedFuncSum. If there are multiple instances of mismatched |
||
| 669 | /// functions, MismatchedFuncSum returns the maximum. |
||
| 670 | Expected<InstrProfRecord> |
||
| 671 | getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, |
||
| 672 | uint64_t *MismatchedFuncSum = nullptr); |
||
| 673 | |||
| 674 | /// Return the memprof record for the function identified by |
||
| 675 | /// llvm::md5(Name). |
||
| 676 | Expected<memprof::MemProfRecord> getMemProfRecord(uint64_t FuncNameHash); |
||
| 677 | |||
| 678 | /// Fill Counts with the profile data for the given function name. |
||
| 679 | Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, |
||
| 680 | std::vector<uint64_t> &Counts); |
||
| 681 | |||
| 682 | /// Return the maximum of all known function counts. |
||
| 683 | /// \c UseCS indicates whether to use the context-sensitive count. |
||
| 684 | uint64_t getMaximumFunctionCount(bool UseCS) { |
||
| 685 | if (UseCS) { |
||
| 686 | assert(CS_Summary && "No context sensitive profile summary"); |
||
| 687 | return CS_Summary->getMaxFunctionCount(); |
||
| 688 | } else { |
||
| 689 | assert(Summary && "No profile summary"); |
||
| 690 | return Summary->getMaxFunctionCount(); |
||
| 691 | } |
||
| 692 | } |
||
| 693 | |||
| 694 | /// Factory method to create an indexed reader. |
||
| 695 | static Expected<std::unique_ptr<IndexedInstrProfReader>> |
||
| 696 | create(const Twine &Path, const Twine &RemappingPath = ""); |
||
| 697 | |||
| 698 | static Expected<std::unique_ptr<IndexedInstrProfReader>> |
||
| 699 | create(std::unique_ptr<MemoryBuffer> Buffer, |
||
| 700 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr); |
||
| 701 | |||
| 702 | // Used for testing purpose only. |
||
| 703 | void setValueProfDataEndianness(support::endianness Endianness) { |
||
| 704 | Index->setValueProfDataEndianness(Endianness); |
||
| 705 | } |
||
| 706 | |||
| 707 | // See description in the base class. This interface is designed |
||
| 708 | // to be used by llvm-profdata (for dumping). Avoid using this when |
||
| 709 | // the client is the compiler. |
||
| 710 | InstrProfSymtab &getSymtab() override; |
||
| 711 | |||
| 712 | /// Return the profile summary. |
||
| 713 | /// \c UseCS indicates whether to use the context-sensitive summary. |
||
| 714 | ProfileSummary &getSummary(bool UseCS) { |
||
| 715 | if (UseCS) { |
||
| 716 | assert(CS_Summary && "No context sensitive summary"); |
||
| 717 | return *CS_Summary; |
||
| 718 | } else { |
||
| 719 | assert(Summary && "No profile summary"); |
||
| 720 | return *Summary; |
||
| 721 | } |
||
| 722 | } |
||
| 723 | |||
| 724 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; |
||
| 725 | Error printBinaryIds(raw_ostream &OS) override; |
||
| 726 | }; |
||
| 727 | |||
| 728 | } // end namespace llvm |
||
| 729 | |||
| 730 | #endif // LLVM_PROFILEDATA_INSTRPROFREADER_H |