Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// Contains utilities for adding indirections and breaking up modules.

//

//===----------------------------------------------------------------------===//

#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H

#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H

#include "llvm/ADT/StringMap.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/ExecutionEngine/JITSymbol.h"

#include "llvm/ExecutionEngine/Orc/Core.h"

#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"

#include "llvm/Support/Error.h"

#include "llvm/Support/Memory.h"

#include "llvm/Support/Process.h"

#include "llvm/Transforms/Utils/ValueMapper.h"

#include <algorithm>

#include <cassert>

#include <cstdint>

#include <functional>

#include <future>

#include <map>

#include <memory>

#include <system_error>

#include <utility>

#include <vector>

namespace llvm {

class Constant;

class Function;

class FunctionType;

class GlobalAlias;

class GlobalVariable;

class Module;

class PointerType;

class Triple;

class Twine;

class Value;

class MCDisassembler;

class MCInstrAnalysis;

namespace jitlink {

class LinkGraph;

class Symbol;

} // namespace jitlink

namespace orc {

/// Base class for pools of compiler re-entry trampolines.

/// These trampolines are callable addresses that save all register state

/// before calling a supplied function to return the trampoline landing

/// address, then restore all state before jumping to that address. They

/// are used by various ORC APIs to support lazy compilation

class TrampolinePool {

public:

  using NotifyLandingResolvedFunction =

      unique_function<void(JITTargetAddress) const>;

  using ResolveLandingFunction = unique_function<void(

      JITTargetAddress TrampolineAddr,

      NotifyLandingResolvedFunction OnLandingResolved) const>;

  virtual ~TrampolinePool();

  /// Get an available trampoline address.

  /// Returns an error if no trampoline can be created.

  Expected<JITTargetAddress> getTrampoline() {

    std::lock_guard<std::mutex> Lock(TPMutex);

    if (AvailableTrampolines.empty()) {

      if (auto Err = grow())

        return std::move(Err);

    }

    assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");

    auto TrampolineAddr = AvailableTrampolines.back();

    AvailableTrampolines.pop_back();

    return TrampolineAddr;

  }

  /// Returns the given trampoline to the pool for re-use.

  void releaseTrampoline(JITTargetAddress TrampolineAddr) {

    std::lock_guard<std::mutex> Lock(TPMutex);

    AvailableTrampolines.push_back(TrampolineAddr);

  }

protected:

  virtual Error grow() = 0;

  std::mutex TPMutex;

  std::vector<JITTargetAddress> AvailableTrampolines;

};

/// A trampoline pool for trampolines within the current process.

template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {

public:

  /// Creates a LocalTrampolinePool with the given RunCallback function.

  /// Returns an error if this function is unable to correctly allocate, write

  /// and protect the resolver code block.

  static Expected<std::unique_ptr<LocalTrampolinePool>>

  Create(ResolveLandingFunction ResolveLanding) {

    Error Err = Error::success();

    auto LTP = std::unique_ptr<LocalTrampolinePool>(

        new LocalTrampolinePool(std::move(ResolveLanding), Err));

    if (Err)

      return std::move(Err);

    return std::move(LTP);

  }

private:

  static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {

    LocalTrampolinePool<ORCABI> *TrampolinePool =

        static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);

    std::promise<JITTargetAddress> LandingAddressP;

    auto LandingAddressF = LandingAddressP.get_future();

    TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),

                                   [&](JITTargetAddress LandingAddress) {

                                     LandingAddressP.set_value(LandingAddress);

                                   });

    return LandingAddressF.get();

  }

  LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)

      : ResolveLanding(std::move(ResolveLanding)) {

    ErrorAsOutParameter _(&Err);

    /// Try to set up the resolver block.

    std::error_code EC;

    ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(

        ORCABI::ResolverCodeSize, nullptr,

        sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));

    if (EC) {

      Err = errorCodeToError(EC);

      return;

    }

    ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),

                              pointerToJITTargetAddress(ResolverBlock.base()),

                              pointerToJITTargetAddress(&reenter),

                              pointerToJITTargetAddress(this));

    EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),

                                          sys::Memory::MF_READ |

                                              sys::Memory::MF_EXEC);

    if (EC) {

      Err = errorCodeToError(EC);

      return;

    }

  }

  Error grow() override {

    assert(AvailableTrampolines.empty() && "Growing prematurely?");

    std::error_code EC;

    auto TrampolineBlock =

        sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(

            sys::Process::getPageSizeEstimate(), nullptr,

            sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));

    if (EC)

      return errorCodeToError(EC);

    unsigned NumTrampolines =

        (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /

        ORCABI::TrampolineSize;

    char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());

    ORCABI::writeTrampolines(

        TrampolineMem, pointerToJITTargetAddress(TrampolineMem),

        pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);

    for (unsigned I = 0; I < NumTrampolines; ++I)

      AvailableTrampolines.push_back(pointerToJITTargetAddress(

          TrampolineMem + (I * ORCABI::TrampolineSize)));

    if (auto EC = sys::Memory::protectMappedMemory(

                    TrampolineBlock.getMemoryBlock(),

                    sys::Memory::MF_READ | sys::Memory::MF_EXEC))

      return errorCodeToError(EC);

    TrampolineBlocks.push_back(std::move(TrampolineBlock));

    return Error::success();

  }

  ResolveLandingFunction ResolveLanding;

  sys::OwningMemoryBlock ResolverBlock;

  std::vector<sys::OwningMemoryBlock> TrampolineBlocks;

};

/// Target-independent base class for compile callback management.

class JITCompileCallbackManager {

public:

  using CompileFunction = std::function<JITTargetAddress()>;

  virtual ~JITCompileCallbackManager() = default;

  /// Reserve a compile callback.

  Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);

  /// Execute the callback for the given trampoline id. Called by the JIT

  ///        to compile functions on demand.

  JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);

protected:

  /// Construct a JITCompileCallbackManager.

  JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,

                            ExecutionSession &ES,

                            JITTargetAddress ErrorHandlerAddress)

      : TP(std::move(TP)), ES(ES),

        CallbacksJD(ES.createBareJITDylib("<Callbacks>")),

        ErrorHandlerAddress(ErrorHandlerAddress) {}

  void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {

    this->TP = std::move(TP);

  }

private:

  std::mutex CCMgrMutex;

  std::unique_ptr<TrampolinePool> TP;

  ExecutionSession &ES;

  JITDylib &CallbacksJD;

  JITTargetAddress ErrorHandlerAddress;

  std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;

  size_t NextCallbackId = 0;

};

/// Manage compile callbacks for in-process JITs.

template <typename ORCABI>

class LocalJITCompileCallbackManager : public JITCompileCallbackManager {

public:

  /// Create a new LocalJITCompileCallbackManager.

  static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>

  Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {

    Error Err = Error::success();

    auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(

        new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));

    if (Err)

      return std::move(Err);

    return std::move(CCMgr);

  }

private:

  /// Construct a InProcessJITCompileCallbackManager.

  /// @param ErrorHandlerAddress The address of an error handler in the target

  ///                            process to be used if a compile callback fails.

  LocalJITCompileCallbackManager(ExecutionSession &ES,

                                 JITTargetAddress ErrorHandlerAddress,

                                 Error &Err)

      : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {

    using NotifyLandingResolvedFunction =

        TrampolinePool::NotifyLandingResolvedFunction;

    ErrorAsOutParameter _(&Err);

    auto TP = LocalTrampolinePool<ORCABI>::Create(

        [this](JITTargetAddress TrampolineAddr,

               NotifyLandingResolvedFunction NotifyLandingResolved) {

          NotifyLandingResolved(executeCompileCallback(TrampolineAddr));

        });

    if (!TP) {

      Err = TP.takeError();

      return;

    }

    setTrampolinePool(std::move(*TP));

  }

};

/// Base class for managing collections of named indirect stubs.

class IndirectStubsManager {

public:

  /// Map type for initializing the manager. See init.

  using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;

  virtual ~IndirectStubsManager() = default;

  /// Create a single stub with the given name, target address and flags.

  virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,

                           JITSymbolFlags StubFlags) = 0;

  /// Create StubInits.size() stubs with the given names, target

  ///        addresses, and flags.

  virtual Error createStubs(const StubInitsMap &StubInits) = 0;

  /// Find the stub with the given name. If ExportedStubsOnly is true,

  ///        this will only return a result if the stub's flags indicate that it

  ///        is exported.

  virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;

  /// Find the implementation-pointer for the stub.

  virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;

  /// Change the value of the implementation pointer for the stub.

  virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;

private:

  virtual void anchor();

};

template <typename ORCABI> class LocalIndirectStubsInfo {

public:

  LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)

      : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}

  static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,

                                                 unsigned PageSize) {

    auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);

    assert((ISAS.StubBytes % PageSize == 0) &&

           "StubBytes is not a page size multiple");

    uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);

    // Allocate memory for stubs and pointers in one call.

    std::error_code EC;

    auto StubsAndPtrsMem =

        sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(

            ISAS.StubBytes + PointerAlloc, nullptr,

            sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));

    if (EC)

      return errorCodeToError(EC);

    sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);

    auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());

    auto PtrBlockAddress =

        pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes;

    ORCABI::writeIndirectStubsBlock(StubsBlockMem,

                                    pointerToJITTargetAddress(StubsBlockMem),

                                    PtrBlockAddress, ISAS.NumStubs);

    if (auto EC = sys::Memory::protectMappedMemory(

            StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC))

      return errorCodeToError(EC);

    return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));

  }

  unsigned getNumStubs() const { return NumStubs; }

  void *getStub(unsigned Idx) const {

    return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;

  }

  void **getPtr(unsigned Idx) const {

    char *PtrsBase =

        static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;

    return reinterpret_cast<void **>(PtrsBase) + Idx;

  }

private:

  unsigned NumStubs = 0;

  sys::OwningMemoryBlock StubsMem;

};

/// IndirectStubsManager implementation for the host architecture, e.g.

///        OrcX86_64. (See OrcArchitectureSupport.h).

template <typename TargetT>

class LocalIndirectStubsManager : public IndirectStubsManager {

public:

  Error createStub(StringRef StubName, JITTargetAddress StubAddr,

                   JITSymbolFlags StubFlags) override {

    std::lock_guard<std::mutex> Lock(StubsMutex);

    if (auto Err = reserveStubs(1))

      return Err;

    createStubInternal(StubName, StubAddr, StubFlags);

    return Error::success();

  }

  Error createStubs(const StubInitsMap &StubInits) override {

    std::lock_guard<std::mutex> Lock(StubsMutex);

    if (auto Err = reserveStubs(StubInits.size()))

      return Err;

    for (const auto &Entry : StubInits)

      createStubInternal(Entry.first(), Entry.second.first,

                         Entry.second.second);

    return Error::success();

  }

  JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {

    std::lock_guard<std::mutex> Lock(StubsMutex);

    auto I = StubIndexes.find(Name);

    if (I == StubIndexes.end())

      return nullptr;

    auto Key = I->second.first;

    void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);

    assert(StubAddr && "Missing stub address");

    auto StubTargetAddr =

        static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));

    auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);

    if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())

      return nullptr;

    return StubSymbol;

  }

  JITEvaluatedSymbol findPointer(StringRef Name) override {

    std::lock_guard<std::mutex> Lock(StubsMutex);

    auto I = StubIndexes.find(Name);

    if (I == StubIndexes.end())

      return nullptr;

    auto Key = I->second.first;

    void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);

    assert(PtrAddr && "Missing pointer address");

    auto PtrTargetAddr =

        static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));

    return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);

  }

  Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {

    using AtomicIntPtr = std::atomic<uintptr_t>;

    std::lock_guard<std::mutex> Lock(StubsMutex);

    auto I = StubIndexes.find(Name);

    assert(I != StubIndexes.end() && "No stub pointer for symbol");

    auto Key = I->second.first;

    AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(

        IndirectStubsInfos[Key.first].getPtr(Key.second));

    *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);

    return Error::success();

  }

private:

  Error reserveStubs(unsigned NumStubs) {

    if (NumStubs <= FreeStubs.size())

      return Error::success();

    unsigned NewStubsRequired = NumStubs - FreeStubs.size();

    unsigned NewBlockId = IndirectStubsInfos.size();

    auto ISI =

        LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);

    if (!ISI)

      return ISI.takeError();

    for (unsigned I = 0; I < ISI->getNumStubs(); ++I)

      FreeStubs.push_back(std::make_pair(NewBlockId, I));

    IndirectStubsInfos.push_back(std::move(*ISI));

    return Error::success();

  }

  void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,

                          JITSymbolFlags StubFlags) {

    auto Key = FreeStubs.back();

    FreeStubs.pop_back();

    *IndirectStubsInfos[Key.first].getPtr(Key.second) =

        jitTargetAddressToPointer<void *>(InitAddr);

    StubIndexes[StubName] = std::make_pair(Key, StubFlags);

  }

  unsigned PageSize = sys::Process::getPageSizeEstimate();

  std::mutex StubsMutex;

  std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;

  using StubKey = std::pair<uint16_t, uint16_t>;

  std::vector<StubKey> FreeStubs;

  StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;

};

/// Create a local compile callback manager.

///

/// The given target triple will determine the ABI, and the given

/// ErrorHandlerAddress will be used by the resulting compile callback

/// manager if a compile callback fails.

Expected<std::unique_ptr<JITCompileCallbackManager>>

createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,

                                  JITTargetAddress ErrorHandlerAddress);

/// Create a local indriect stubs manager builder.

///

/// The given target triple will determine the ABI.

std::function<std::unique_ptr<IndirectStubsManager>()>

createLocalIndirectStubsManagerBuilder(const Triple &T);

/// Build a function pointer of FunctionType with the given constant

///        address.

///

///   Usage example: Turn a trampoline address into a function pointer constant

/// for use in a stub.

Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);

/// Create a function pointer with the given type, name, and initializer

///        in the given Module.

GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,

                                  Constant *Initializer);

/// Turn a function declaration into a stub function that makes an

///        indirect call using the given function pointer.

void makeStub(Function &F, Value &ImplPointer);

/// Promotes private symbols to global hidden, and renames to prevent clashes

/// with other promoted symbols. The same SymbolPromoter instance should be

/// used for all symbols to be added to a single JITDylib.

class SymbolLinkagePromoter {

public:

  /// Promote symbols in the given module. Returns the set of global values

  /// that have been renamed/promoted.

  std::vector<GlobalValue *> operator()(Module &M);

private:

  unsigned NextId = 0;

};

/// Clone a function declaration into a new module.

///

///   This function can be used as the first step towards creating a callback

/// stub (see makeStub), or moving a function body (see moveFunctionBody).

///

///   If the VMap argument is non-null, a mapping will be added between F and

/// the new declaration, and between each of F's arguments and the new

/// declaration's arguments. This map can then be passed in to moveFunction to

/// move the function body if required. Note: When moving functions between

/// modules with these utilities, all decls should be cloned (and added to a

/// single VMap) before any bodies are moved. This will ensure that references

/// between functions all refer to the versions in the new module.

Function *cloneFunctionDecl(Module &Dst, const Function &F,

                            ValueToValueMapTy *VMap = nullptr);

/// Move the body of function 'F' to a cloned function declaration in a

///        different module (See related cloneFunctionDecl).

///

///   If the target function declaration is not supplied via the NewF parameter

/// then it will be looked up via the VMap.

///

///   This will delete the body of function 'F' from its original parent module,

/// but leave its declaration.

void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,

                      ValueMaterializer *Materializer = nullptr,

                      Function *NewF = nullptr);

/// Clone a global variable declaration into a new module.

GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,

                                        ValueToValueMapTy *VMap = nullptr);

/// Move global variable GV from its parent module to cloned global

///        declaration in a different module.

///

///   If the target global declaration is not supplied via the NewGV parameter

/// then it will be looked up via the VMap.

///

///   This will delete the initializer of GV from its original parent module,

/// but leave its declaration.

void moveGlobalVariableInitializer(GlobalVariable &OrigGV,

                                   ValueToValueMapTy &VMap,

                                   ValueMaterializer *Materializer = nullptr,

                                   GlobalVariable *NewGV = nullptr);

/// Clone a global alias declaration into a new module.

GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,

                                  ValueToValueMapTy &VMap);

/// Clone module flags metadata into the destination module.

void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,

                              ValueToValueMapTy &VMap);

/// Introduce relocations to \p Sym in its own definition if there are any

/// pointers formed via PC-relative address that do not already have a

/// relocation.

///

/// This is useful when introducing indirection via a stub function at link time

/// without compiler support. If a function pointer is formed without a

/// relocation, e.g. in the definition of \c foo

///

/// \code

/// _foo:

///   leaq -7(%rip), rax # form pointer to _foo without relocation

/// _bar:

///   leaq (%rip), %rax  # uses X86_64_RELOC_SIGNED to '_foo'

/// \endcode

///

/// the pointer to \c _foo computed by \c _foo and \c _bar may differ if we

/// introduce a stub for _foo. If the pointer is used as a key, this may be

/// observable to the program. This pass will attempt to introduce the missing

/// "self-relocation" on the leaq instruction.

///

/// This is based on disassembly and should be considered "best effort". It may

/// silently fail to add relocations.

Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,

                                                   jitlink::LinkGraph &G,

                                                   MCDisassembler &Disassembler,

                                                   MCInstrAnalysis &MIA);

} // end namespace orc

} // end namespace llvm

#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H