Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H

#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H

#include "clang/Basic/LLVM.h"

#include "clang/Lex/DependencyDirectivesScanner.h"

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/StringMap.h"

#include "llvm/Support/Allocator.h"

#include "llvm/Support/ErrorOr.h"

#include "llvm/Support/VirtualFileSystem.h"

#include <mutex>

#include <optional>

namespace clang {

namespace tooling {

namespace dependencies {

using DependencyDirectivesTy =

    SmallVector<dependency_directives_scan::Directive, 20>;

/// Contents and directive tokens of a cached file entry. Single instance can

/// be shared between multiple entries.

struct CachedFileContents {

  CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Contents)

      : Original(std::move(Contents)), DepDirectives(nullptr) {}

  /// Owning storage for the original contents.

  std::unique_ptr<llvm::MemoryBuffer> Original;

  /// The mutex that must be locked before mutating directive tokens.

  std::mutex ValueLock;

  SmallVector<dependency_directives_scan::Token, 10> DepDirectiveTokens;

  /// Accessor to the directive tokens that's atomic to avoid data races.

  /// \p CachedFileContents has ownership of the pointer.

  std::atomic<const std::optional<DependencyDirectivesTy> *> DepDirectives;

  ~CachedFileContents() { delete DepDirectives.load(); }

};

/// An in-memory representation of a file system entity that is of interest to

/// the dependency scanning filesystem.

///

/// It represents one of the following:

/// - opened file with contents and a stat value,

/// - opened file with contents, directive tokens and a stat value,

/// - directory entry with its stat value,

/// - filesystem error.

///

/// Single instance of this class can be shared across different filenames (e.g.

/// a regular file and a symlink). For this reason the status filename is empty

/// and is only materialized by \c EntryRef that knows the requested filename.

class CachedFileSystemEntry {

public:

  /// Creates an entry without contents: either a filesystem error or

  /// a directory with stat value.

  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat)

      : MaybeStat(std::move(Stat)), Contents(nullptr) {

    clearStatName();

  }

  /// Creates an entry representing a file with contents.

  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat,

                        CachedFileContents *Contents)

      : MaybeStat(std::move(Stat)), Contents(std::move(Contents)) {

    clearStatName();

  }

  /// \returns True if the entry is a filesystem error.

  bool isError() const { return !MaybeStat; }

  /// \returns True if the current entry represents a directory.

  bool isDirectory() const { return !isError() && MaybeStat->isDirectory(); }

  /// \returns Original contents of the file.

  StringRef getOriginalContents() const {

    assert(!isError() && "error");

    assert(!MaybeStat->isDirectory() && "not a file");

    assert(Contents && "contents not initialized");

    return Contents->Original->getBuffer();

  }

  /// \returns The scanned preprocessor directive tokens of the file that are

  /// used to speed up preprocessing, if available.

  std::optional<ArrayRef<dependency_directives_scan::Directive>>

  getDirectiveTokens() const {

    assert(!isError() && "error");

    assert(!isDirectory() && "not a file");

    assert(Contents && "contents not initialized");

    if (auto *Directives = Contents->DepDirectives.load()) {

      if (Directives->has_value())

        return ArrayRef<dependency_directives_scan::Directive>(**Directives);

    }

    return std::nullopt;

  }

  /// \returns The error.

  std::error_code getError() const { return MaybeStat.getError(); }

  /// \returns The entry status with empty filename.

  llvm::vfs::Status getStatus() const {

    assert(!isError() && "error");

    assert(MaybeStat->getName().empty() && "stat name must be empty");

    return *MaybeStat;

  }

  /// \returns The unique ID of the entry.

  llvm::sys::fs::UniqueID getUniqueID() const {

    assert(!isError() && "error");

    return MaybeStat->getUniqueID();

  }

  /// \returns The data structure holding both contents and directive tokens.

  CachedFileContents *getCachedContents() const {

    assert(!isError() && "error");

    assert(!isDirectory() && "not a file");

    return Contents;

  }

private:

  void clearStatName() {

    if (MaybeStat)

      MaybeStat = llvm::vfs::Status::copyWithNewName(*MaybeStat, "");

  }

  /// Either the filesystem error or status of the entry.

  /// The filename is empty and only materialized by \c EntryRef.

  llvm::ErrorOr<llvm::vfs::Status> MaybeStat;

  /// Non-owning pointer to the file contents.

  ///

  /// We're using pointer here to keep the size of this class small. Instances

  /// representing directories and filesystem errors don't hold any contents

  /// anyway.

  CachedFileContents *Contents;

};

/// This class is a shared cache, that caches the 'stat' and 'open' calls to the

/// underlying real file system, and the scanned preprocessor directives of

/// files.

///

/// It is sharded based on the hash of the key to reduce the lock contention for

/// the worker threads.

class DependencyScanningFilesystemSharedCache {

public:

  struct CacheShard {

    /// The mutex that needs to be locked before mutation of any member.

    mutable std::mutex CacheLock;

    /// Map from filenames to cached entries.

    llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>

        EntriesByFilename;

    /// Map from unique IDs to cached entries.

    llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *>

        EntriesByUID;

    /// The backing storage for cached entries.

    llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> EntryStorage;

    /// The backing storage for cached contents.

    llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;

    /// Returns entry associated with the filename or nullptr if none is found.

    const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const;

    /// Returns entry associated with the unique ID or nullptr if none is found.

    const CachedFileSystemEntry *

    findEntryByUID(llvm::sys::fs::UniqueID UID) const;

    /// Returns entry associated with the filename if there is some. Otherwise,

    /// constructs new one with the given status, associates it with the

    /// filename and returns the result.

    const CachedFileSystemEntry &

    getOrEmplaceEntryForFilename(StringRef Filename,

                                 llvm::ErrorOr<llvm::vfs::Status> Stat);

    /// Returns entry associated with the unique ID if there is some. Otherwise,

    /// constructs new one with the given status and contents, associates it

    /// with the unique ID and returns the result.

    const CachedFileSystemEntry &

    getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,

                            std::unique_ptr<llvm::MemoryBuffer> Contents);

    /// Returns entry associated with the filename if there is some. Otherwise,

    /// associates the given entry with the filename and returns it.

    const CachedFileSystemEntry &

    getOrInsertEntryForFilename(StringRef Filename,

                                const CachedFileSystemEntry &Entry);

  };

  DependencyScanningFilesystemSharedCache();

  /// Returns shard for the given key.

  CacheShard &getShardForFilename(StringRef Filename) const;

  CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;

private:

  std::unique_ptr<CacheShard[]> CacheShards;

  unsigned NumShards;

};

/// This class is a local cache, that caches the 'stat' and 'open' calls to the

/// underlying real file system.

class DependencyScanningFilesystemLocalCache {

  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;

public:

  /// Returns entry associated with the filename or nullptr if none is found.

  const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {

    auto It = Cache.find(Filename);

    return It == Cache.end() ? nullptr : It->getValue();

  }

  /// Associates the given entry with the filename and returns the given entry

  /// pointer (for convenience).

  const CachedFileSystemEntry &

  insertEntryForFilename(StringRef Filename,

                         const CachedFileSystemEntry &Entry) {

    const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second;

    assert(InsertedEntry == &Entry && "entry already present");

    return *InsertedEntry;

  }

};

/// Reference to a CachedFileSystemEntry.

/// If the underlying entry is an opened file, this wrapper returns the file

/// contents and the scanned preprocessor directives.

class EntryRef {

  /// The filename used to access this entry.

  std::string Filename;

  /// The underlying cached entry.

  const CachedFileSystemEntry &Entry;

public:

  EntryRef(StringRef Name, const CachedFileSystemEntry &Entry)

      : Filename(Name), Entry(Entry) {}

  llvm::vfs::Status getStatus() const {

    llvm::vfs::Status Stat = Entry.getStatus();

    if (!Stat.isDirectory())

      Stat = llvm::vfs::Status::copyWithNewSize(Stat, getContents().size());

    return llvm::vfs::Status::copyWithNewName(Stat, Filename);

  }

  bool isError() const { return Entry.isError(); }

  bool isDirectory() const { return Entry.isDirectory(); }

  /// If the cached entry represents an error, promotes it into `ErrorOr`.

  llvm::ErrorOr<EntryRef> unwrapError() const {

    if (isError())

      return Entry.getError();

    return *this;

  }

  StringRef getContents() const { return Entry.getOriginalContents(); }

  std::optional<ArrayRef<dependency_directives_scan::Directive>>

  getDirectiveTokens() const {

    return Entry.getDirectiveTokens();

  }

};

/// A virtual file system optimized for the dependency discovery.

///

/// It is primarily designed to work with source files whose contents was

/// preprocessed to remove any tokens that are unlikely to affect the dependency

/// computation.

///

/// This is not a thread safe VFS. A single instance is meant to be used only in

/// one thread. Multiple instances are allowed to service multiple threads

/// running in parallel.

class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {

public:

  DependencyScanningWorkerFilesystem(

      DependencyScanningFilesystemSharedCache &SharedCache,

      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)

      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {}

  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;

  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>

  openFileForRead(const Twine &Path) override;

  /// Returns entry for the given filename.

  ///

  /// Attempts to use the local and shared caches first, then falls back to

  /// using the underlying filesystem.

  llvm::ErrorOr<EntryRef>

  getOrCreateFileSystemEntry(StringRef Filename,

                             bool DisableDirectivesScanning = false);

private:

  /// Check whether the file should be scanned for preprocessor directives.

  bool shouldScanForDirectives(StringRef Filename);

  /// For a filename that's not yet associated with any entry in the caches,

  /// uses the underlying filesystem to either look up the entry based in the

  /// shared cache indexed by unique ID, or creates new entry from scratch.

  llvm::ErrorOr<const CachedFileSystemEntry &>

  computeAndStoreResult(StringRef Filename);

  /// Scan for preprocessor directives for the given entry if necessary and

  /// returns a wrapper object with reference semantics.

  EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry,

                                        StringRef Filename, bool Disable);

  /// Represents a filesystem entry that has been stat-ed (and potentially read)

  /// and that's about to be inserted into the cache as `CachedFileSystemEntry`.

  struct TentativeEntry {

    llvm::vfs::Status Status;

    std::unique_ptr<llvm::MemoryBuffer> Contents;

    TentativeEntry(llvm::vfs::Status Status,

                   std::unique_ptr<llvm::MemoryBuffer> Contents = nullptr)

        : Status(std::move(Status)), Contents(std::move(Contents)) {}

  };

  /// Reads file at the given path. Enforces consistency between the file size

  /// in status and size of read contents.

  llvm::ErrorOr<TentativeEntry> readFile(StringRef Filename);

  /// Returns entry associated with the unique ID of the given tentative entry

  /// if there is some in the shared cache. Otherwise, constructs new one,

  /// associates it with the unique ID and returns the result.

  const CachedFileSystemEntry &

  getOrEmplaceSharedEntryForUID(TentativeEntry TEntry);

  /// Returns entry associated with the filename or nullptr if none is found.

  ///

  /// Returns entry from local cache if there is some. Otherwise, if the entry

  /// is found in the shared cache, writes it through the local cache and

  /// returns it. Otherwise returns nullptr.

  const CachedFileSystemEntry *

  findEntryByFilenameWithWriteThrough(StringRef Filename);

  /// Returns entry associated with the unique ID in the shared cache or nullptr

  /// if none is found.

  const CachedFileSystemEntry *

  findSharedEntryByUID(llvm::vfs::Status Stat) const {

    return SharedCache.getShardForUID(Stat.getUniqueID())

        .findEntryByUID(Stat.getUniqueID());

  }

  /// Associates the given entry with the filename in the local cache and

  /// returns it.

  const CachedFileSystemEntry &

  insertLocalEntryForFilename(StringRef Filename,

                              const CachedFileSystemEntry &Entry) {

    return LocalCache.insertEntryForFilename(Filename, Entry);

  }

  /// Returns entry associated with the filename in the shared cache if there is

  /// some. Otherwise, constructs new one with the given error code, associates

  /// it with the filename and returns the result.

  const CachedFileSystemEntry &

  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {

    return SharedCache.getShardForFilename(Filename)

        .getOrEmplaceEntryForFilename(Filename, EC);

  }

  /// Returns entry associated with the filename in the shared cache if there is

  /// some. Otherwise, associates the given entry with the filename and returns

  /// it.

  const CachedFileSystemEntry &

  getOrInsertSharedEntryForFilename(StringRef Filename,

                                    const CachedFileSystemEntry &Entry) {

    return SharedCache.getShardForFilename(Filename)

        .getOrInsertEntryForFilename(Filename, Entry);

  }

  void printImpl(raw_ostream &OS, PrintType Type,

                 unsigned IndentLevel) const override {

    printIndent(OS, IndentLevel);

    OS << "DependencyScanningFilesystem\n";

    getUnderlyingFS().print(OS, Type, IndentLevel + 1);

  }

  /// The global cache shared between worker threads.

  DependencyScanningFilesystemSharedCache &SharedCache;

  /// The local cache is used by the worker thread to cache file system queries

  /// locally instead of querying the global cache every time.

  DependencyScanningFilesystemLocalCache LocalCache;

};

} // end namespace dependencies

} // end namespace tooling

} // end namespace clang

#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H