Details | Last modification | View Log | RSS feed
| Rev | Author | Line No. | Line | 
|---|---|---|---|
| 14 | pmbaty | 1 | //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===// | 
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | // | ||
| 9 | // This file contains support for reading profiling data for instrumentation | ||
| 10 | // based PGO and coverage. | ||
| 11 | // | ||
| 12 | //===----------------------------------------------------------------------===// | ||
| 13 | |||
| 14 | #ifndef LLVM_PROFILEDATA_INSTRPROFREADER_H | ||
| 15 | #define LLVM_PROFILEDATA_INSTRPROFREADER_H | ||
| 16 | |||
| 17 | #include "llvm/ADT/ArrayRef.h" | ||
| 18 | #include "llvm/ADT/StringRef.h" | ||
| 19 | #include "llvm/IR/ProfileSummary.h" | ||
| 20 | #include "llvm/Object/BuildID.h" | ||
| 21 | #include "llvm/ProfileData/InstrProf.h" | ||
| 22 | #include "llvm/ProfileData/InstrProfCorrelator.h" | ||
| 23 | #include "llvm/ProfileData/MemProf.h" | ||
| 24 | #include "llvm/Support/Endian.h" | ||
| 25 | #include "llvm/Support/Error.h" | ||
| 26 | #include "llvm/Support/LineIterator.h" | ||
| 27 | #include "llvm/Support/MathExtras.h" | ||
| 28 | #include "llvm/Support/MemoryBuffer.h" | ||
| 29 | #include "llvm/Support/OnDiskHashTable.h" | ||
| 30 | #include "llvm/Support/SwapByteOrder.h" | ||
| 31 | #include <algorithm> | ||
| 32 | #include <cassert> | ||
| 33 | #include <cstddef> | ||
| 34 | #include <cstdint> | ||
| 35 | #include <iterator> | ||
| 36 | #include <memory> | ||
| 37 | #include <utility> | ||
| 38 | #include <vector> | ||
| 39 | |||
| 40 | namespace llvm { | ||
| 41 | |||
| 42 | class InstrProfReader; | ||
| 43 | |||
| 44 | /// A file format agnostic iterator over profiling data. | ||
| 45 | template <class record_type = NamedInstrProfRecord, | ||
| 46 | class reader_type = InstrProfReader> | ||
| 47 | class InstrProfIterator { | ||
| 48 | public: | ||
| 49 | using iterator_category = std::input_iterator_tag; | ||
| 50 | using value_type = record_type; | ||
| 51 | using difference_type = std::ptrdiff_t; | ||
| 52 | using pointer = value_type *; | ||
| 53 | using reference = value_type &; | ||
| 54 | |||
| 55 | private: | ||
| 56 | reader_type *Reader = nullptr; | ||
| 57 |   value_type Record; | ||
| 58 | |||
| 59 | void increment() { | ||
| 60 | if (Error E = Reader->readNextRecord(Record)) { | ||
| 61 |       // Handle errors in the reader. | ||
| 62 | InstrProfError::take(std::move(E)); | ||
| 63 | *this = InstrProfIterator(); | ||
| 64 |     } | ||
| 65 |   } | ||
| 66 | |||
| 67 | public: | ||
| 68 | InstrProfIterator() = default; | ||
| 69 | InstrProfIterator(reader_type *Reader) : Reader(Reader) { increment(); } | ||
| 70 | |||
| 71 | InstrProfIterator &operator++() { | ||
| 72 | increment(); | ||
| 73 | return *this; | ||
| 74 |   } | ||
| 75 | bool operator==(const InstrProfIterator &RHS) const { | ||
| 76 | return Reader == RHS.Reader; | ||
| 77 |   } | ||
| 78 | bool operator!=(const InstrProfIterator &RHS) const { | ||
| 79 | return Reader != RHS.Reader; | ||
| 80 |   } | ||
| 81 | value_type &operator*() { return Record; } | ||
| 82 | value_type *operator->() { return &Record; } | ||
| 83 | }; | ||
| 84 | |||
| 85 | /// Base class and interface for reading profiling data of any known instrprof | ||
| 86 | /// format. Provides an iterator over NamedInstrProfRecords. | ||
| 87 | class InstrProfReader { | ||
| 88 | instrprof_error LastError = instrprof_error::success; | ||
| 89 | std::string LastErrorMsg; | ||
| 90 | |||
| 91 | public: | ||
| 92 | InstrProfReader() = default; | ||
| 93 | virtual ~InstrProfReader() = default; | ||
| 94 | |||
| 95 |   /// Read the header.  Required before reading first record. | ||
| 96 | virtual Error readHeader() = 0; | ||
| 97 | |||
| 98 |   /// Read a single record. | ||
| 99 | virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0; | ||
| 100 | |||
| 101 |   /// Read a list of binary ids. | ||
| 102 | virtual Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) { | ||
| 103 | return success(); | ||
| 104 |   } | ||
| 105 | |||
| 106 |   /// Print binary ids. | ||
| 107 | virtual Error printBinaryIds(raw_ostream &OS) { return success(); }; | ||
| 108 | |||
| 109 |   /// Iterator over profile data. | ||
| 110 | InstrProfIterator<> begin() { return InstrProfIterator<>(this); } | ||
| 111 | InstrProfIterator<> end() { return InstrProfIterator<>(); } | ||
| 112 | |||
| 113 |   /// Return the profile version. | ||
| 114 | virtual uint64_t getVersion() const = 0; | ||
| 115 | |||
| 116 | virtual bool isIRLevelProfile() const = 0; | ||
| 117 | |||
| 118 | virtual bool hasCSIRLevelProfile() const = 0; | ||
| 119 | |||
| 120 | virtual bool instrEntryBBEnabled() const = 0; | ||
| 121 | |||
| 122 |   /// Return true if we must provide debug info to create PGO profiles. | ||
| 123 | virtual bool useDebugInfoCorrelate() const { return false; } | ||
| 124 | |||
| 125 |   /// Return true if the profile has single byte counters representing coverage. | ||
| 126 | virtual bool hasSingleByteCoverage() const = 0; | ||
| 127 | |||
| 128 |   /// Return true if the profile only instruments function entries. | ||
| 129 | virtual bool functionEntryOnly() const = 0; | ||
| 130 | |||
| 131 |   /// Return true if profile includes a memory profile. | ||
| 132 | virtual bool hasMemoryProfile() const = 0; | ||
| 133 | |||
| 134 |   /// Returns a BitsetEnum describing the attributes of the profile. To check | ||
| 135 |   /// individual attributes prefer using the helpers above. | ||
| 136 | virtual InstrProfKind getProfileKind() const = 0; | ||
| 137 | |||
| 138 |   /// Return the PGO symtab. There are three different readers: | ||
| 139 |   /// Raw, Text, and Indexed profile readers. The first two types | ||
| 140 |   /// of readers are used only by llvm-profdata tool, while the indexed | ||
| 141 |   /// profile reader is also used by llvm-cov tool and the compiler ( | ||
| 142 |   /// backend or frontend). Since creating PGO symtab can create | ||
| 143 |   /// significant runtime and memory overhead (as it touches data | ||
| 144 |   /// for the whole program), InstrProfSymtab for the indexed profile | ||
| 145 |   /// reader should be created on demand and it is recommended to be | ||
| 146 |   /// only used for dumping purpose with llvm-proftool, not with the | ||
| 147 |   /// compiler. | ||
| 148 | virtual InstrProfSymtab &getSymtab() = 0; | ||
| 149 | |||
| 150 |   /// Compute the sum of counts and return in Sum. | ||
| 151 | void accumulateCounts(CountSumOrPercent &Sum, bool IsCS); | ||
| 152 | |||
| 153 | protected: | ||
| 154 | std::unique_ptr<InstrProfSymtab> Symtab; | ||
| 155 | |||
| 156 |   /// Set the current error and return same. | ||
| 157 | Error error(instrprof_error Err, const std::string &ErrMsg = "") { | ||
| 158 | LastError = Err; | ||
| 159 | LastErrorMsg = ErrMsg; | ||
| 160 | if (Err == instrprof_error::success) | ||
| 161 | return Error::success(); | ||
| 162 | return make_error<InstrProfError>(Err, ErrMsg); | ||
| 163 |   } | ||
| 164 | |||
| 165 | Error error(Error &&E) { | ||
| 166 | handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { | ||
| 167 | LastError = IPE.get(); | ||
| 168 | LastErrorMsg = IPE.getMessage(); | ||
| 169 | }); | ||
| 170 | return make_error<InstrProfError>(LastError, LastErrorMsg); | ||
| 171 |   } | ||
| 172 | |||
| 173 |   /// Clear the current error and return a successful one. | ||
| 174 | Error success() { return error(instrprof_error::success); } | ||
| 175 | |||
| 176 | public: | ||
| 177 |   /// Return true if the reader has finished reading the profile data. | ||
| 178 | bool isEOF() { return LastError == instrprof_error::eof; } | ||
| 179 | |||
| 180 |   /// Return true if the reader encountered an error reading profiling data. | ||
| 181 | bool hasError() { return LastError != instrprof_error::success && !isEOF(); } | ||
| 182 | |||
| 183 |   /// Get the current error. | ||
| 184 | Error getError() { | ||
| 185 | if (hasError()) | ||
| 186 | return make_error<InstrProfError>(LastError, LastErrorMsg); | ||
| 187 | return Error::success(); | ||
| 188 |   } | ||
| 189 | |||
| 190 |   /// Factory method to create an appropriately typed reader for the given | ||
| 191 |   /// instrprof file. | ||
| 192 | static Expected<std::unique_ptr<InstrProfReader>> | ||
| 193 | create(const Twine &Path, const InstrProfCorrelator *Correlator = nullptr); | ||
| 194 | |||
| 195 | static Expected<std::unique_ptr<InstrProfReader>> | ||
| 196 | create(std::unique_ptr<MemoryBuffer> Buffer, | ||
| 197 | const InstrProfCorrelator *Correlator = nullptr); | ||
| 198 | }; | ||
| 199 | |||
| 200 | /// Reader for the simple text based instrprof format. | ||
| 201 | /// | ||
| 202 | /// This format is a simple text format that's suitable for test data. Records | ||
| 203 | /// are separated by one or more blank lines, and record fields are separated by | ||
| 204 | /// new lines. | ||
| 205 | /// | ||
| 206 | /// Each record consists of a function name, a function hash, a number of | ||
| 207 | /// counters, and then each counter value, in that order. | ||
| 208 | class TextInstrProfReader : public InstrProfReader { | ||
| 209 | private: | ||
| 210 |   /// The profile data file contents. | ||
| 211 | std::unique_ptr<MemoryBuffer> DataBuffer; | ||
| 212 |   /// Iterator over the profile data. | ||
| 213 |   line_iterator Line; | ||
| 214 |   /// The attributes of the current profile. | ||
| 215 | InstrProfKind ProfileKind = InstrProfKind::Unknown; | ||
| 216 | |||
| 217 | Error readValueProfileData(InstrProfRecord &Record); | ||
| 218 | |||
| 219 | public: | ||
| 220 | TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_) | ||
| 221 | : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} | ||
| 222 | TextInstrProfReader(const TextInstrProfReader &) = delete; | ||
| 223 | TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; | ||
| 224 | |||
| 225 |   /// Return true if the given buffer is in text instrprof format. | ||
| 226 | static bool hasFormat(const MemoryBuffer &Buffer); | ||
| 227 | |||
| 228 |   // Text format does not have version, so return 0. | ||
| 229 | uint64_t getVersion() const override { return 0; } | ||
| 230 | |||
| 231 | bool isIRLevelProfile() const override { | ||
| 232 | return static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation); | ||
| 233 |   } | ||
| 234 | |||
| 235 | bool hasCSIRLevelProfile() const override { | ||
| 236 | return static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive); | ||
| 237 |   } | ||
| 238 | |||
| 239 | bool instrEntryBBEnabled() const override { | ||
| 240 | return static_cast<bool>(ProfileKind & | ||
| 241 | InstrProfKind::FunctionEntryInstrumentation); | ||
| 242 |   } | ||
| 243 | |||
| 244 | bool hasSingleByteCoverage() const override { | ||
| 245 | return static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage); | ||
| 246 |   } | ||
| 247 | |||
| 248 | bool functionEntryOnly() const override { | ||
| 249 | return static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly); | ||
| 250 |   } | ||
| 251 | |||
| 252 | bool hasMemoryProfile() const override { | ||
| 253 |     // TODO: Add support for text format memory profiles. | ||
| 254 | return false; | ||
| 255 |   } | ||
| 256 | |||
| 257 | InstrProfKind getProfileKind() const override { return ProfileKind; } | ||
| 258 | |||
| 259 |   /// Read the header. | ||
| 260 | Error readHeader() override; | ||
| 261 | |||
| 262 |   /// Read a single record. | ||
| 263 | Error readNextRecord(NamedInstrProfRecord &Record) override; | ||
| 264 | |||
| 265 | InstrProfSymtab &getSymtab() override { | ||
| 266 | assert(Symtab); | ||
| 267 | return *Symtab; | ||
| 268 |   } | ||
| 269 | }; | ||
| 270 | |||
| 271 | /// Reader for the raw instrprof binary format from runtime. | ||
| 272 | /// | ||
| 273 | /// This format is a raw memory dump of the instrumentation-based profiling data | ||
| 274 | /// from the runtime.  It has no index. | ||
| 275 | /// | ||
| 276 | /// Templated on the unsigned type whose size matches pointers on the platform | ||
| 277 | /// that wrote the profile. | ||
| 278 | template <class IntPtrT> | ||
| 279 | class RawInstrProfReader : public InstrProfReader { | ||
| 280 | private: | ||
| 281 |   /// The profile data file contents. | ||
| 282 | std::unique_ptr<MemoryBuffer> DataBuffer; | ||
| 283 |   /// If available, this hold the ProfileData array used to correlate raw | ||
| 284 |   /// instrumentation data to their functions. | ||
| 285 | const InstrProfCorrelatorImpl<IntPtrT> *Correlator; | ||
| 286 | bool ShouldSwapBytes; | ||
| 287 |   // The value of the version field of the raw profile data header. The lower 56 | ||
| 288 |   // bits specifies the format version and the most significant 8 bits specify | ||
| 289 |   // the variant types of the profile. | ||
| 290 | uint64_t Version; | ||
| 291 | uint64_t CountersDelta; | ||
| 292 | uint64_t NamesDelta; | ||
| 293 | const RawInstrProf::ProfileData<IntPtrT> *Data; | ||
| 294 | const RawInstrProf::ProfileData<IntPtrT> *DataEnd; | ||
| 295 | const char *CountersStart; | ||
| 296 | const char *CountersEnd; | ||
| 297 | const char *NamesStart; | ||
| 298 | const char *NamesEnd; | ||
| 299 |   // After value profile is all read, this pointer points to | ||
| 300 |   // the header of next profile data (if exists) | ||
| 301 | const uint8_t *ValueDataStart; | ||
| 302 | uint32_t ValueKindLast; | ||
| 303 | uint32_t CurValueDataSize; | ||
| 304 | |||
| 305 |   /// Total size of binary ids. | ||
| 306 | uint64_t BinaryIdsSize{0}; | ||
| 307 |   /// Start address of binary id length and data pairs. | ||
| 308 | const uint8_t *BinaryIdsStart; | ||
| 309 | |||
| 310 | public: | ||
| 311 | RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer, | ||
| 312 | const InstrProfCorrelator *Correlator) | ||
| 313 | : DataBuffer(std::move(DataBuffer)), | ||
| 314 | Correlator(dyn_cast_or_null<const InstrProfCorrelatorImpl<IntPtrT>>( | ||
| 315 | Correlator)) {} | ||
| 316 | RawInstrProfReader(const RawInstrProfReader &) = delete; | ||
| 317 | RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; | ||
| 318 | |||
| 319 | static bool hasFormat(const MemoryBuffer &DataBuffer); | ||
| 320 | Error readHeader() override; | ||
| 321 | Error readNextRecord(NamedInstrProfRecord &Record) override; | ||
| 322 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; | ||
| 323 | Error printBinaryIds(raw_ostream &OS) override; | ||
| 324 | |||
| 325 | uint64_t getVersion() const override { return Version; } | ||
| 326 | |||
| 327 | bool isIRLevelProfile() const override { | ||
| 328 | return (Version & VARIANT_MASK_IR_PROF) != 0; | ||
| 329 |   } | ||
| 330 | |||
| 331 | bool hasCSIRLevelProfile() const override { | ||
| 332 | return (Version & VARIANT_MASK_CSIR_PROF) != 0; | ||
| 333 |   } | ||
| 334 | |||
| 335 | bool instrEntryBBEnabled() const override { | ||
| 336 | return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; | ||
| 337 |   } | ||
| 338 | |||
| 339 | bool useDebugInfoCorrelate() const override { | ||
| 340 | return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; | ||
| 341 |   } | ||
| 342 | |||
| 343 | bool hasSingleByteCoverage() const override { | ||
| 344 | return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; | ||
| 345 |   } | ||
| 346 | |||
| 347 | bool functionEntryOnly() const override { | ||
| 348 | return (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; | ||
| 349 |   } | ||
| 350 | |||
| 351 | bool hasMemoryProfile() const override { | ||
| 352 |     // Memory profiles have a separate raw format, so this should never be set. | ||
| 353 | assert(!(Version & VARIANT_MASK_MEMPROF)); | ||
| 354 | return false; | ||
| 355 |   } | ||
| 356 | |||
| 357 |   /// Returns a BitsetEnum describing the attributes of the raw instr profile. | ||
| 358 | InstrProfKind getProfileKind() const override; | ||
| 359 | |||
| 360 | InstrProfSymtab &getSymtab() override { | ||
| 361 | assert(Symtab.get()); | ||
| 362 | return *Symtab.get(); | ||
| 363 |   } | ||
| 364 | |||
| 365 | private: | ||
| 366 | Error createSymtab(InstrProfSymtab &Symtab); | ||
| 367 | Error readNextHeader(const char *CurrentPos); | ||
| 368 | Error readHeader(const RawInstrProf::Header &Header); | ||
| 369 | |||
| 370 | template <class IntT> IntT swap(IntT Int) const { | ||
| 371 | return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int; | ||
| 372 |   } | ||
| 373 | |||
| 374 | support::endianness getDataEndianness() const { | ||
| 375 | support::endianness HostEndian = getHostEndianness(); | ||
| 376 | if (!ShouldSwapBytes) | ||
| 377 | return HostEndian; | ||
| 378 | if (HostEndian == support::little) | ||
| 379 | return support::big; | ||
| 380 |     else | ||
| 381 | return support::little; | ||
| 382 |   } | ||
| 383 | |||
| 384 | inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) { | ||
| 385 | return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); | ||
| 386 |   } | ||
| 387 | |||
| 388 | Error readName(NamedInstrProfRecord &Record); | ||
| 389 | Error readFuncHash(NamedInstrProfRecord &Record); | ||
| 390 | Error readRawCounts(InstrProfRecord &Record); | ||
| 391 | Error readValueProfilingData(InstrProfRecord &Record); | ||
| 392 | bool atEnd() const { return Data == DataEnd; } | ||
| 393 | |||
| 394 | void advanceData() { | ||
| 395 |     // `CountersDelta` is a constant zero when using debug info correlation. | ||
| 396 | if (!Correlator) { | ||
| 397 |       // The initial CountersDelta is the in-memory address difference between | ||
| 398 |       // the data and counts sections: | ||
| 399 |       // start(__llvm_prf_cnts) - start(__llvm_prf_data) | ||
| 400 |       // As we advance to the next record, we maintain the correct CountersDelta | ||
| 401 |       // with respect to the next record. | ||
| 402 | CountersDelta -= sizeof(*Data); | ||
| 403 |     } | ||
| 404 | Data++; | ||
| 405 | ValueDataStart += CurValueDataSize; | ||
| 406 |   } | ||
| 407 | |||
| 408 | const char *getNextHeaderPos() const { | ||
| 409 | assert(atEnd()); | ||
| 410 | return (const char *)ValueDataStart; | ||
| 411 |   } | ||
| 412 | |||
| 413 | StringRef getName(uint64_t NameRef) const { | ||
| 414 | return Symtab->getFuncName(swap(NameRef)); | ||
| 415 |   } | ||
| 416 | |||
| 417 | int getCounterTypeSize() const { | ||
| 418 | return hasSingleByteCoverage() ? sizeof(uint8_t) : sizeof(uint64_t); | ||
| 419 |   } | ||
| 420 | }; | ||
| 421 | |||
| 422 | using RawInstrProfReader32 = RawInstrProfReader<uint32_t>; | ||
| 423 | using RawInstrProfReader64 = RawInstrProfReader<uint64_t>; | ||
| 424 | |||
| 425 | namespace IndexedInstrProf { | ||
| 426 | |||
| 427 | enum class HashT : uint32_t; | ||
| 428 | |||
| 429 | } // end namespace IndexedInstrProf | ||
| 430 | |||
| 431 | /// Trait for lookups into the on-disk hash table for the binary instrprof | ||
| 432 | /// format. | ||
| 433 | class InstrProfLookupTrait { | ||
| 434 | std::vector<NamedInstrProfRecord> DataBuffer; | ||
| 435 | IndexedInstrProf::HashT HashType; | ||
| 436 | unsigned FormatVersion; | ||
| 437 |   // Endianness of the input value profile data. | ||
| 438 |   // It should be LE by default, but can be changed | ||
| 439 |   // for testing purpose. | ||
| 440 | support::endianness ValueProfDataEndianness = support::little; | ||
| 441 | |||
| 442 | public: | ||
| 443 | InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion) | ||
| 444 | : HashType(HashType), FormatVersion(FormatVersion) {} | ||
| 445 | |||
| 446 | using data_type = ArrayRef<NamedInstrProfRecord>; | ||
| 447 | |||
| 448 | using internal_key_type = StringRef; | ||
| 449 | using external_key_type = StringRef; | ||
| 450 | using hash_value_type = uint64_t; | ||
| 451 | using offset_type = uint64_t; | ||
| 452 | |||
| 453 | static bool EqualKey(StringRef A, StringRef B) { return A == B; } | ||
| 454 | static StringRef GetInternalKey(StringRef K) { return K; } | ||
| 455 | static StringRef GetExternalKey(StringRef K) { return K; } | ||
| 456 | |||
| 457 | hash_value_type ComputeHash(StringRef K); | ||
| 458 | |||
| 459 | static std::pair<offset_type, offset_type> | ||
| 460 | ReadKeyDataLength(const unsigned char *&D) { | ||
| 461 | using namespace support; | ||
| 462 | |||
| 463 | offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D); | ||
| 464 | offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D); | ||
| 465 | return std::make_pair(KeyLen, DataLen); | ||
| 466 |   } | ||
| 467 | |||
| 468 | StringRef ReadKey(const unsigned char *D, offset_type N) { | ||
| 469 | return StringRef((const char *)D, N); | ||
| 470 |   } | ||
| 471 | |||
| 472 | bool readValueProfilingData(const unsigned char *&D, | ||
| 473 | const unsigned char *const End); | ||
| 474 | data_type ReadData(StringRef K, const unsigned char *D, offset_type N); | ||
| 475 | |||
| 476 |   // Used for testing purpose only. | ||
| 477 | void setValueProfDataEndianness(support::endianness Endianness) { | ||
| 478 | ValueProfDataEndianness = Endianness; | ||
| 479 |   } | ||
| 480 | }; | ||
| 481 | |||
| 482 | struct InstrProfReaderIndexBase { | ||
| 483 | virtual ~InstrProfReaderIndexBase() = default; | ||
| 484 | |||
| 485 |   // Read all the profile records with the same key pointed to the current | ||
| 486 |   // iterator. | ||
| 487 | virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0; | ||
| 488 | |||
| 489 |   // Read all the profile records with the key equal to FuncName | ||
| 490 | virtual Error getRecords(StringRef FuncName, | ||
| 491 | ArrayRef<NamedInstrProfRecord> &Data) = 0; | ||
| 492 | virtual void advanceToNextKey() = 0; | ||
| 493 | virtual bool atEnd() const = 0; | ||
| 494 | virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; | ||
| 495 | virtual uint64_t getVersion() const = 0; | ||
| 496 | virtual bool isIRLevelProfile() const = 0; | ||
| 497 | virtual bool hasCSIRLevelProfile() const = 0; | ||
| 498 | virtual bool instrEntryBBEnabled() const = 0; | ||
| 499 | virtual bool hasSingleByteCoverage() const = 0; | ||
| 500 | virtual bool functionEntryOnly() const = 0; | ||
| 501 | virtual bool hasMemoryProfile() const = 0; | ||
| 502 | virtual InstrProfKind getProfileKind() const = 0; | ||
| 503 | virtual Error populateSymtab(InstrProfSymtab &) = 0; | ||
| 504 | }; | ||
| 505 | |||
| 506 | using OnDiskHashTableImplV3 = | ||
| 507 | OnDiskIterableChainedHashTable<InstrProfLookupTrait>; | ||
| 508 | |||
| 509 | using MemProfRecordHashTable = | ||
| 510 | OnDiskIterableChainedHashTable<memprof::RecordLookupTrait>; | ||
| 511 | using MemProfFrameHashTable = | ||
| 512 | OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>; | ||
| 513 | |||
| 514 | template <typename HashTableImpl> | ||
| 515 | class InstrProfReaderItaniumRemapper; | ||
| 516 | |||
| 517 | template <typename HashTableImpl> | ||
| 518 | class InstrProfReaderIndex : public InstrProfReaderIndexBase { | ||
| 519 | private: | ||
| 520 | std::unique_ptr<HashTableImpl> HashTable; | ||
| 521 | typename HashTableImpl::data_iterator RecordIterator; | ||
| 522 | uint64_t FormatVersion; | ||
| 523 | |||
| 524 | friend class InstrProfReaderItaniumRemapper<HashTableImpl>; | ||
| 525 | |||
| 526 | public: | ||
| 527 | InstrProfReaderIndex(const unsigned char *Buckets, | ||
| 528 | const unsigned char *const Payload, | ||
| 529 | const unsigned char *const Base, | ||
| 530 | IndexedInstrProf::HashT HashType, uint64_t Version); | ||
| 531 | ~InstrProfReaderIndex() override = default; | ||
| 532 | |||
| 533 | Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override; | ||
| 534 |   Error getRecords(StringRef FuncName, | ||
| 535 | ArrayRef<NamedInstrProfRecord> &Data) override; | ||
| 536 | void advanceToNextKey() override { RecordIterator++; } | ||
| 537 | |||
| 538 | bool atEnd() const override { | ||
| 539 | return RecordIterator == HashTable->data_end(); | ||
| 540 |   } | ||
| 541 | |||
| 542 | void setValueProfDataEndianness(support::endianness Endianness) override { | ||
| 543 | HashTable->getInfoObj().setValueProfDataEndianness(Endianness); | ||
| 544 |   } | ||
| 545 | |||
| 546 | uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } | ||
| 547 | |||
| 548 | bool isIRLevelProfile() const override { | ||
| 549 | return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; | ||
| 550 |   } | ||
| 551 | |||
| 552 | bool hasCSIRLevelProfile() const override { | ||
| 553 | return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; | ||
| 554 |   } | ||
| 555 | |||
| 556 | bool instrEntryBBEnabled() const override { | ||
| 557 | return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0; | ||
| 558 |   } | ||
| 559 | |||
| 560 | bool hasSingleByteCoverage() const override { | ||
| 561 | return (FormatVersion & VARIANT_MASK_BYTE_COVERAGE) != 0; | ||
| 562 |   } | ||
| 563 | |||
| 564 | bool functionEntryOnly() const override { | ||
| 565 | return (FormatVersion & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; | ||
| 566 |   } | ||
| 567 | |||
| 568 | bool hasMemoryProfile() const override { | ||
| 569 | return (FormatVersion & VARIANT_MASK_MEMPROF) != 0; | ||
| 570 |   } | ||
| 571 | |||
| 572 | InstrProfKind getProfileKind() const override; | ||
| 573 | |||
| 574 | Error populateSymtab(InstrProfSymtab &Symtab) override { | ||
| 575 | return Symtab.create(HashTable->keys()); | ||
| 576 |   } | ||
| 577 | }; | ||
| 578 | |||
| 579 | /// Name matcher supporting fuzzy matching of symbol names to names in profiles. | ||
| 580 | class InstrProfReaderRemapper { | ||
| 581 | public: | ||
| 582 | virtual ~InstrProfReaderRemapper() = default; | ||
| 583 | virtual Error populateRemappings() { return Error::success(); } | ||
| 584 | virtual Error getRecords(StringRef FuncName, | ||
| 585 | ArrayRef<NamedInstrProfRecord> &Data) = 0; | ||
| 586 | }; | ||
| 587 | |||
| 588 | /// Reader for the indexed binary instrprof format. | ||
| 589 | class IndexedInstrProfReader : public InstrProfReader { | ||
| 590 | private: | ||
| 591 |   /// The profile data file contents. | ||
| 592 | std::unique_ptr<MemoryBuffer> DataBuffer; | ||
| 593 |   /// The profile remapping file contents. | ||
| 594 | std::unique_ptr<MemoryBuffer> RemappingBuffer; | ||
| 595 |   /// The index into the profile data. | ||
| 596 | std::unique_ptr<InstrProfReaderIndexBase> Index; | ||
| 597 |   /// The profile remapping file contents. | ||
| 598 | std::unique_ptr<InstrProfReaderRemapper> Remapper; | ||
| 599 |   /// Profile summary data. | ||
| 600 | std::unique_ptr<ProfileSummary> Summary; | ||
| 601 |   /// Context sensitive profile summary data. | ||
| 602 | std::unique_ptr<ProfileSummary> CS_Summary; | ||
| 603 |   /// MemProf profile schema (if available). | ||
| 604 | memprof::MemProfSchema Schema; | ||
| 605 |   /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName). | ||
| 606 | std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable; | ||
| 607 |   /// MemProf frame profile data on-disk indexed via frame id. | ||
| 608 | std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable; | ||
| 609 |   /// Total size of binary ids. | ||
| 610 | uint64_t BinaryIdsSize{0}; | ||
| 611 |   /// Start address of binary id length and data pairs. | ||
| 612 | const uint8_t *BinaryIdsStart = nullptr; | ||
| 613 | |||
| 614 |   // Index to the current record in the record array. | ||
| 615 | unsigned RecordIndex; | ||
| 616 | |||
| 617 |   // Read the profile summary. Return a pointer pointing to one byte past the | ||
| 618 |   // end of the summary data if it exists or the input \c Cur. | ||
| 619 |   // \c UseCS indicates whether to use the context-sensitive profile summary. | ||
| 620 | const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, | ||
| 621 | const unsigned char *Cur, bool UseCS); | ||
| 622 | |||
| 623 | public: | ||
| 624 |   IndexedInstrProfReader( | ||
| 625 | std::unique_ptr<MemoryBuffer> DataBuffer, | ||
| 626 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr) | ||
| 627 | : DataBuffer(std::move(DataBuffer)), | ||
| 628 | RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {} | ||
| 629 | IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; | ||
| 630 | IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; | ||
| 631 | |||
| 632 |   /// Return the profile version. | ||
| 633 | uint64_t getVersion() const override { return Index->getVersion(); } | ||
| 634 | bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } | ||
| 635 | bool hasCSIRLevelProfile() const override { | ||
| 636 | return Index->hasCSIRLevelProfile(); | ||
| 637 |   } | ||
| 638 | |||
| 639 | bool instrEntryBBEnabled() const override { | ||
| 640 | return Index->instrEntryBBEnabled(); | ||
| 641 |   } | ||
| 642 | |||
| 643 | bool hasSingleByteCoverage() const override { | ||
| 644 | return Index->hasSingleByteCoverage(); | ||
| 645 |   } | ||
| 646 | |||
| 647 | bool functionEntryOnly() const override { return Index->functionEntryOnly(); } | ||
| 648 | |||
| 649 | bool hasMemoryProfile() const override { return Index->hasMemoryProfile(); } | ||
| 650 | |||
| 651 |   /// Returns a BitsetEnum describing the attributes of the indexed instr | ||
| 652 |   /// profile. | ||
| 653 | InstrProfKind getProfileKind() const override { | ||
| 654 | return Index->getProfileKind(); | ||
| 655 |   } | ||
| 656 | |||
| 657 |   /// Return true if the given buffer is in an indexed instrprof format. | ||
| 658 | static bool hasFormat(const MemoryBuffer &DataBuffer); | ||
| 659 | |||
| 660 |   /// Read the file header. | ||
| 661 | Error readHeader() override; | ||
| 662 |   /// Read a single record. | ||
| 663 | Error readNextRecord(NamedInstrProfRecord &Record) override; | ||
| 664 | |||
| 665 |   /// Return the NamedInstrProfRecord associated with FuncName and FuncHash. | ||
| 666 |   /// When return a hash_mismatch error and MismatchedFuncSum is not nullptr, | ||
| 667 |   /// the sum of all counters in the mismatched function will be set to | ||
| 668 |   /// MismatchedFuncSum. If there are multiple instances of mismatched | ||
| 669 |   /// functions, MismatchedFuncSum returns the maximum. | ||
| 670 | Expected<InstrProfRecord> | ||
| 671 | getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, | ||
| 672 | uint64_t *MismatchedFuncSum = nullptr); | ||
| 673 | |||
| 674 |   /// Return the memprof record for the function identified by | ||
| 675 |   /// llvm::md5(Name). | ||
| 676 | Expected<memprof::MemProfRecord> getMemProfRecord(uint64_t FuncNameHash); | ||
| 677 | |||
| 678 |   /// Fill Counts with the profile data for the given function name. | ||
| 679 | Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, | ||
| 680 | std::vector<uint64_t> &Counts); | ||
| 681 | |||
| 682 |   /// Return the maximum of all known function counts. | ||
| 683 |   /// \c UseCS indicates whether to use the context-sensitive count. | ||
| 684 | uint64_t getMaximumFunctionCount(bool UseCS) { | ||
| 685 | if (UseCS) { | ||
| 686 | assert(CS_Summary && "No context sensitive profile summary"); | ||
| 687 | return CS_Summary->getMaxFunctionCount(); | ||
| 688 | } else { | ||
| 689 | assert(Summary && "No profile summary"); | ||
| 690 | return Summary->getMaxFunctionCount(); | ||
| 691 |     } | ||
| 692 |   } | ||
| 693 | |||
| 694 |   /// Factory method to create an indexed reader. | ||
| 695 | static Expected<std::unique_ptr<IndexedInstrProfReader>> | ||
| 696 | create(const Twine &Path, const Twine &RemappingPath = ""); | ||
| 697 | |||
| 698 | static Expected<std::unique_ptr<IndexedInstrProfReader>> | ||
| 699 | create(std::unique_ptr<MemoryBuffer> Buffer, | ||
| 700 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr); | ||
| 701 | |||
| 702 |   // Used for testing purpose only. | ||
| 703 | void setValueProfDataEndianness(support::endianness Endianness) { | ||
| 704 | Index->setValueProfDataEndianness(Endianness); | ||
| 705 |   } | ||
| 706 | |||
| 707 |   // See description in the base class. This interface is designed | ||
| 708 |   // to be used by llvm-profdata (for dumping). Avoid using this when | ||
| 709 |   // the client is the compiler. | ||
| 710 | InstrProfSymtab &getSymtab() override; | ||
| 711 | |||
| 712 |   /// Return the profile summary. | ||
| 713 |   /// \c UseCS indicates whether to use the context-sensitive summary. | ||
| 714 | ProfileSummary &getSummary(bool UseCS) { | ||
| 715 | if (UseCS) { | ||
| 716 | assert(CS_Summary && "No context sensitive summary"); | ||
| 717 | return *CS_Summary; | ||
| 718 | } else { | ||
| 719 | assert(Summary && "No profile summary"); | ||
| 720 | return *Summary; | ||
| 721 |     } | ||
| 722 |   } | ||
| 723 | |||
| 724 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; | ||
| 725 | Error printBinaryIds(raw_ostream &OS) override; | ||
| 726 | }; | ||
| 727 | |||
| 728 | } // end namespace llvm | ||
| 729 | |||
| 730 | #endif // LLVM_PROFILEDATA_INSTRPROFREADER_H |