WebSVN – QNX 8.QNX8 LLVM/Clang compiler suite – Blame – //llvm-build/x86_64/include/clang/Lex/Preprocessor.h

Rev	Author	Line No.	Line
14	pmbaty	1	//===- Preprocessor.h - C Language Family Preprocessor ----------- C++ --===//
		2	//
		3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
		4	// See https://llvm.org/LICENSE.txt for license information.
		5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
		6	//
		7	//===----------------------------------------------------------------------===//
		8	//
		9	/// \file
		10	/// Defines the clang::Preprocessor interface.
		11	//
		12	//===----------------------------------------------------------------------===//
		13
		14	#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
		15	#define LLVM_CLANG_LEX_PREPROCESSOR_H
		16
		17	#include "clang/Basic/Diagnostic.h"
		18	#include "clang/Basic/DiagnosticIDs.h"
		19	#include "clang/Basic/IdentifierTable.h"
		20	#include "clang/Basic/LLVM.h"
		21	#include "clang/Basic/LangOptions.h"
		22	#include "clang/Basic/Module.h"
		23	#include "clang/Basic/SourceLocation.h"
		24	#include "clang/Basic/SourceManager.h"
		25	#include "clang/Basic/TokenKinds.h"
		26	#include "clang/Lex/HeaderSearch.h"
		27	#include "clang/Lex/Lexer.h"
		28	#include "clang/Lex/MacroInfo.h"
		29	#include "clang/Lex/ModuleLoader.h"
		30	#include "clang/Lex/ModuleMap.h"
		31	#include "clang/Lex/PPCallbacks.h"
		32	#include "clang/Lex/Token.h"
		33	#include "clang/Lex/TokenLexer.h"
		34	#include "llvm/ADT/ArrayRef.h"
		35	#include "llvm/ADT/DenseMap.h"
		36	#include "llvm/ADT/FoldingSet.h"
		37	#include "llvm/ADT/FunctionExtras.h"
		38	#include "llvm/ADT/PointerUnion.h"
		39	#include "llvm/ADT/STLExtras.h"
		40	#include "llvm/ADT/SmallPtrSet.h"
		41	#include "llvm/ADT/SmallVector.h"
		42	#include "llvm/ADT/StringRef.h"
		43	#include "llvm/ADT/TinyPtrVector.h"
		44	#include "llvm/ADT/iterator_range.h"
		45	#include "llvm/Support/Allocator.h"
		46	#include "llvm/Support/Casting.h"
		47	#include "llvm/Support/Registry.h"
		48	#include <cassert>
		49	#include <cstddef>
		50	#include <cstdint>
		51	#include <map>
		52	#include <memory>
		53	#include <optional>
		54	#include <string>
		55	#include <utility>
		56	#include <vector>
		57
		58	namespace llvm {
		59
		60	template<unsigned InternalLen> class SmallString;
		61
		62	} // namespace llvm
		63
		64	namespace clang {
		65
		66	class CodeCompletionHandler;
		67	class CommentHandler;
		68	class DirectoryEntry;
		69	class EmptylineHandler;
		70	class ExternalPreprocessorSource;
		71	class FileEntry;
		72	class FileManager;
		73	class HeaderSearch;
		74	class MacroArgs;
		75	class PragmaHandler;
		76	class PragmaNamespace;
		77	class PreprocessingRecord;
		78	class PreprocessorLexer;
		79	class PreprocessorOptions;
		80	class ScratchBuffer;
		81	class TargetInfo;
		82
		83	namespace Builtin {
		84	class Context;
		85	}
		86
		87	/// Stores token information for comparing actual tokens with
		88	/// predefined values. Only handles simple tokens and identifiers.
		89	class TokenValue {
		90	tok::TokenKind Kind;
		91	IdentifierInfo *II;
		92
		93	public:
		94	TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
		95	assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
		96	assert(Kind != tok::identifier &&
		97	"Identifiers should be created by TokenValue(IdentifierInfo *)");
		98	assert(!tok::isLiteral(Kind) && "Literals are not supported.");
		99	assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
		100	}
		101
		102	TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
		103
		104	bool operator==(const Token &Tok) const {
		105	return Tok.getKind() == Kind &&
		106	(!II \|\| II == Tok.getIdentifierInfo());
		107	}
		108	};
		109
		110	/// Context in which macro name is used.
		111	enum MacroUse {
		112	// other than #define or #undef
		113	MU_Other = 0,
		114
		115	// macro name specified in #define
		116	MU_Define = 1,
		117
		118	// macro name specified in #undef
		119	MU_Undef = 2
		120	};
		121
		122	/// Engages in a tight little dance with the lexer to efficiently
		123	/// preprocess tokens.
		124	///
		125	/// Lexers know only about tokens within a single source file, and don't
		126	/// know anything about preprocessor-level issues like the \#include stack,
		127	/// token expansion, etc.
		128	class Preprocessor {
		129	friend class VAOptDefinitionContext;
		130	friend class VariadicMacroScopeGuard;
		131
		132	llvm::unique_function<void(const clang::Token &)> OnToken;
		133	std::shared_ptr<PreprocessorOptions> PPOpts;
		134	DiagnosticsEngine *Diags;
		135	LangOptions &LangOpts;
		136	const TargetInfo *Target = nullptr;
		137	const TargetInfo *AuxTarget = nullptr;
		138	FileManager &FileMgr;
		139	SourceManager &SourceMgr;
		140	std::unique_ptr<ScratchBuffer> ScratchBuf;
		141	HeaderSearch &HeaderInfo;
		142	ModuleLoader &TheModuleLoader;
		143
		144	/// External source of macros.
		145	ExternalPreprocessorSource *ExternalSource;
		146
		147	/// A BumpPtrAllocator object used to quickly allocate and release
		148	/// objects internal to the Preprocessor.
		149	llvm::BumpPtrAllocator BP;
		150
		151	/// Identifiers for builtin macros and other builtins.
		152	IdentifierInfo Ident__LINE__, Ident__FILE__; // __LINE__, __FILE__
		153	IdentifierInfo Ident__DATE__, Ident__TIME__; // __DATE__, __TIME__
		154	IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
		155	IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
		156	IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
		157	IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
		158	IdentifierInfo *Ident__COUNTER__; // __COUNTER__
		159	IdentifierInfo Ident_Pragma, Ident__pragma; // _Pragma, __pragma
		160	IdentifierInfo *Ident__identifier; // __identifier
		161	IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
		162	IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
		163	IdentifierInfo *Ident__has_feature; // __has_feature
		164	IdentifierInfo *Ident__has_extension; // __has_extension
		165	IdentifierInfo *Ident__has_builtin; // __has_builtin
		166	IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
		167	IdentifierInfo *Ident__has_attribute; // __has_attribute
		168	IdentifierInfo *Ident__has_include; // __has_include
		169	IdentifierInfo *Ident__has_include_next; // __has_include_next
		170	IdentifierInfo *Ident__has_warning; // __has_warning
		171	IdentifierInfo *Ident__is_identifier; // __is_identifier
		172	IdentifierInfo *Ident__building_module; // __building_module
		173	IdentifierInfo *Ident__MODULE__; // __MODULE__
		174	IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
		175	IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
		176	IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
		177	IdentifierInfo *Ident__is_target_arch; // __is_target_arch
		178	IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
		179	IdentifierInfo *Ident__is_target_os; // __is_target_os
		180	IdentifierInfo *Ident__is_target_environment; // __is_target_environment
		181	IdentifierInfo *Ident__is_target_variant_os;
		182	IdentifierInfo *Ident__is_target_variant_environment;
		183	IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
		184
		185	// Weak, only valid (and set) while InMacroArgs is true.
		186	Token* ArgMacro;
		187
		188	SourceLocation DATELoc, TIMELoc;
		189
		190	// FEM_UnsetOnCommandLine means that an explicit evaluation method was
		191	// not specified on the command line. The target is queried to set the
		192	// default evaluation method.
		193	LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
		194	LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
		195
		196	// The most recent pragma location where the floating point evaluation
		197	// method was modified. This is used to determine whether the
		198	// 'pragma clang fp eval_method' was used whithin the current scope.
		199	SourceLocation LastFPEvalPragmaLocation;
		200
		201	LangOptions::FPEvalMethodKind TUFPEvalMethod =
		202	LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
		203
		204	// Next __COUNTER__ value, starts at 0.
		205	unsigned CounterValue = 0;
		206
		207	enum {
		208	/// Maximum depth of \#includes.
		209	MaxAllowedIncludeStackDepth = 200
		210	};
		211
		212	// State that is set before the preprocessor begins.
		213	bool KeepComments : 1;
		214	bool KeepMacroComments : 1;
		215	bool SuppressIncludeNotFoundError : 1;
		216
		217	// State that changes while the preprocessor runs:
		218	bool InMacroArgs : 1; // True if parsing fn macro invocation args.
		219
		220	/// Whether the preprocessor owns the header search object.
		221	bool OwnsHeaderSearch : 1;
		222
		223	/// True if macro expansion is disabled.
		224	bool DisableMacroExpansion : 1;
		225
		226	/// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
		227	/// when parsing preprocessor directives.
		228	bool MacroExpansionInDirectivesOverride : 1;
		229
		230	class ResetMacroExpansionHelper;
		231
		232	/// Whether we have already loaded macros from the external source.
		233	mutable bool ReadMacrosFromExternalSource : 1;
		234
		235	/// True if pragmas are enabled.
		236	bool PragmasEnabled : 1;
		237
		238	/// True if the current build action is a preprocessing action.
		239	bool PreprocessedOutput : 1;
		240
		241	/// True if we are currently preprocessing a #if or #elif directive
		242	bool ParsingIfOrElifDirective;
		243
		244	/// True if we are pre-expanding macro arguments.
		245	bool InMacroArgPreExpansion;
		246
		247	/// Mapping/lookup information for all identifiers in
		248	/// the program, including program keywords.
		249	mutable IdentifierTable Identifiers;
		250
		251	/// This table contains all the selectors in the program.
		252	///
		253	/// Unlike IdentifierTable above, this table isn't populated by the
		254	/// preprocessor. It is declared/expanded here because its role/lifetime is
		255	/// conceptually similar to the IdentifierTable. In addition, the current
		256	/// control flow (in clang::ParseAST()), make it convenient to put here.
		257	///
		258	/// FIXME: Make sure the lifetime of Identifiers/Selectors isn't tied to
		259	/// the lifetime of the preprocessor.
		260	SelectorTable Selectors;
		261
		262	/// Information about builtins.
		263	std::unique_ptr<Builtin::Context> BuiltinInfo;
		264
		265	/// Tracks all of the pragmas that the client registered
		266	/// with this preprocessor.
		267	std::unique_ptr<PragmaNamespace> PragmaHandlers;
		268
		269	/// Pragma handlers of the original source is stored here during the
		270	/// parsing of a model file.
		271	std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
		272
		273	/// Tracks all of the comment handlers that the client registered
		274	/// with this preprocessor.
		275	std::vector<CommentHandler *> CommentHandlers;
		276
		277	/// Empty line handler.
		278	EmptylineHandler *Emptyline = nullptr;
		279
		280	public:
		281	/// The kind of translation unit we are processing.
		282	const TranslationUnitKind TUKind;
		283
		284	private:
		285	/// The code-completion handler.
		286	CodeCompletionHandler *CodeComplete = nullptr;
		287
		288	/// The file that we're performing code-completion for, if any.
		289	const FileEntry *CodeCompletionFile = nullptr;
		290
		291	/// The offset in file for the code-completion point.
		292	unsigned CodeCompletionOffset = 0;
		293
		294	/// The location for the code-completion point. This gets instantiated
		295	/// when the CodeCompletionFile gets \#include'ed for preprocessing.
		296	SourceLocation CodeCompletionLoc;
		297
		298	/// The start location for the file of the code-completion point.
		299	///
		300	/// This gets instantiated when the CodeCompletionFile gets \#include'ed
		301	/// for preprocessing.
		302	SourceLocation CodeCompletionFileLoc;
		303
		304	/// The source location of the \c import contextual keyword we just
		305	/// lexed, if any.
		306	SourceLocation ModuleImportLoc;
		307
		308	/// The import path for named module that we're currently processing.
		309	SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> NamedModuleImportPath;
		310
		311	/// Whether the import is an `@import` or a standard c++ modules import.
		312	bool IsAtImport = false;
		313
		314	/// Whether the last token we lexed was an '@'.
		315	bool LastTokenWasAt = false;
		316
		317	/// A position within a C++20 import-seq.
		318	class StdCXXImportSeq {
		319	public:
		320	enum State : int {
		321	// Positive values represent a number of unclosed brackets.
		322	AtTopLevel = 0,
		323	AfterTopLevelTokenSeq = -1,
		324	AfterExport = -2,
		325	AfterImportSeq = -3,
		326	};
		327
		328	StdCXXImportSeq(State S) : S(S) {}
		329
		330	/// Saw any kind of open bracket.
		331	void handleOpenBracket() {
		332	S = static_cast<State>(std::max<int>(S, 0) + 1);
		333	}
		334	/// Saw any kind of close bracket other than '}'.
		335	void handleCloseBracket() {
		336	S = static_cast<State>(std::max<int>(S, 1) - 1);
		337	}
		338	/// Saw a close brace.
		339	void handleCloseBrace() {
		340	handleCloseBracket();
		341	if (S == AtTopLevel && !AfterHeaderName)
		342	S = AfterTopLevelTokenSeq;
		343	}
		344	/// Saw a semicolon.
		345	void handleSemi() {
		346	if (atTopLevel()) {
		347	S = AfterTopLevelTokenSeq;
		348	AfterHeaderName = false;
		349	}
		350	}
		351
		352	/// Saw an 'export' identifier.
		353	void handleExport() {
		354	if (S == AfterTopLevelTokenSeq)
		355	S = AfterExport;
		356	else if (S <= 0)
		357	S = AtTopLevel;
		358	}
		359	/// Saw an 'import' identifier.
		360	void handleImport() {
		361	if (S == AfterTopLevelTokenSeq \|\| S == AfterExport)
		362	S = AfterImportSeq;
		363	else if (S <= 0)
		364	S = AtTopLevel;
		365	}
		366
		367	/// Saw a 'header-name' token; do not recognize any more 'import' tokens
		368	/// until we reach a top-level semicolon.
		369	void handleHeaderName() {
		370	if (S == AfterImportSeq)
		371	AfterHeaderName = true;
		372	handleMisc();
		373	}
		374
		375	/// Saw any other token.
		376	void handleMisc() {
		377	if (S <= 0)
		378	S = AtTopLevel;
		379	}
		380
		381	bool atTopLevel() { return S <= 0; }
		382	bool afterImportSeq() { return S == AfterImportSeq; }
		383	bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
		384
		385	private:
		386	State S;
		387	/// Whether we're in the pp-import-suffix following the header-name in a
		388	/// pp-import. If so, a close-brace is not sufficient to end the
		389	/// top-level-token-seq of an import-seq.
		390	bool AfterHeaderName = false;
		391	};
		392
		393	/// Our current position within a C++20 import-seq.
		394	StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
		395
		396	/// Track whether we are in a Global Module Fragment
		397	class TrackGMF {
		398	public:
		399	enum GMFState : int {
		400	GMFActive = 1,
		401	MaybeGMF = 0,
		402	BeforeGMFIntroducer = -1,
		403	GMFAbsentOrEnded = -2,
		404	};
		405
		406	TrackGMF(GMFState S) : S(S) {}
		407
		408	/// Saw a semicolon.
		409	void handleSemi() {
		410	// If it is immediately after the first instance of the module keyword,
		411	// then that introduces the GMF.
		412	if (S == MaybeGMF)
		413	S = GMFActive;
		414	}
		415
		416	/// Saw an 'export' identifier.
		417	void handleExport() {
		418	// The presence of an 'export' keyword always ends or excludes a GMF.
		419	S = GMFAbsentOrEnded;
		420	}
		421
		422	/// Saw an 'import' identifier.
		423	void handleImport(bool AfterTopLevelTokenSeq) {
		424	// If we see this before any 'module' kw, then we have no GMF.
		425	if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
		426	S = GMFAbsentOrEnded;
		427	}
		428
		429	/// Saw a 'module' identifier.
		430	void handleModule(bool AfterTopLevelTokenSeq) {
		431	// This was the first module identifier and not preceded by any token
		432	// that would exclude a GMF. It could begin a GMF, but only if directly
		433	// followed by a semicolon.
		434	if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
		435	S = MaybeGMF;
		436	else
		437	S = GMFAbsentOrEnded;
		438	}
		439
		440	/// Saw any other token.
		441	void handleMisc() {
		442	// We saw something other than ; after the 'module' kw, so not a GMF.
		443	if (S == MaybeGMF)
		444	S = GMFAbsentOrEnded;
		445	}
		446
		447	bool inGMF() { return S == GMFActive; }
		448
		449	private:
		450	/// Track the transitions into and out of a Global Module Fragment,
		451	/// if one is present.
		452	GMFState S;
		453	};
		454
		455	TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
		456
		457	/// Track the status of the c++20 module decl.
		458	///
		459	/// module-declaration:
		460	/// 'export'[opt] 'module' module-name module-partition[opt]
		461	/// attribute-specifier-seq[opt] ';'
		462	///
		463	/// module-name:
		464	/// module-name-qualifier[opt] identifier
		465	///
		466	/// module-partition:
		467	/// ':' module-name-qualifier[opt] identifier
		468	///
		469	/// module-name-qualifier:
		470	/// identifier '.'
		471	/// module-name-qualifier identifier '.'
		472	///
		473	/// Transition state:
		474	///
		475	/// NotAModuleDecl --- export ---> FoundExport
		476	/// NotAModuleDecl --- module ---> ImplementationCandidate
		477	/// FoundExport --- module ---> InterfaceCandidate
		478	/// ImplementationCandidate --- Identifier ---> ImplementationCandidate
		479	/// ImplementationCandidate --- period ---> ImplementationCandidate
		480	/// ImplementationCandidate --- colon ---> ImplementationCandidate
		481	/// InterfaceCandidate --- Identifier ---> InterfaceCandidate
		482	/// InterfaceCandidate --- period ---> InterfaceCandidate
		483	/// InterfaceCandidate --- colon ---> InterfaceCandidate
		484	/// ImplementationCandidate --- Semi ---> NamedModuleImplementation
		485	/// NamedModuleInterface --- Semi ---> NamedModuleInterface
		486	/// NamedModuleImplementation --- Anything ---> NamedModuleImplementation
		487	/// NamedModuleInterface --- Anything ---> NamedModuleInterface
		488	///
		489	/// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
		490	/// soon since we don't support any module attributes yet.
		491	class ModuleDeclSeq {
		492	enum ModuleDeclState : int {
		493	NotAModuleDecl,
		494	FoundExport,
		495	InterfaceCandidate,
		496	ImplementationCandidate,
		497	NamedModuleInterface,
		498	NamedModuleImplementation,
		499	};
		500
		501	public:
		502	ModuleDeclSeq() : State(NotAModuleDecl) {}
		503
		504	void handleExport() {
		505	if (State == NotAModuleDecl)
		506	State = FoundExport;
		507	else if (!isNamedModule())
		508	reset();
		509	}
		510
		511	void handleModule() {
		512	if (State == FoundExport)
		513	State = InterfaceCandidate;
		514	else if (State == NotAModuleDecl)
		515	State = ImplementationCandidate;
		516	else if (!isNamedModule())
		517	reset();
		518	}
		519
		520	void handleIdentifier(IdentifierInfo *Identifier) {
		521	if (isModuleCandidate() && Identifier)
		522	Name += Identifier->getName().str();
		523	else if (!isNamedModule())
		524	reset();
		525	}
		526
		527	void handleColon() {
		528	if (isModuleCandidate())
		529	Name += ":";
		530	else if (!isNamedModule())
		531	reset();
		532	}
		533
		534	void handlePeriod() {
		535	if (isModuleCandidate())
		536	Name += ".";
		537	else if (!isNamedModule())
		538	reset();
		539	}
		540
		541	void handleSemi() {
		542	if (!Name.empty() && isModuleCandidate()) {
		543	if (State == InterfaceCandidate)
		544	State = NamedModuleInterface;
		545	else if (State == ImplementationCandidate)
		546	State = NamedModuleImplementation;
		547	else
		548	llvm_unreachable("Unimaged ModuleDeclState.");
		549	} else if (!isNamedModule())
		550	reset();
		551	}
		552
		553	void handleMisc() {
		554	if (!isNamedModule())
		555	reset();
		556	}
		557
		558	bool isModuleCandidate() const {
		559	return State == InterfaceCandidate \|\| State == ImplementationCandidate;
		560	}
		561
		562	bool isNamedModule() const {
		563	return State == NamedModuleInterface \|\|
		564	State == NamedModuleImplementation;
		565	}
		566
		567	bool isNamedInterface() const { return State == NamedModuleInterface; }
		568
		569	bool isImplementationUnit() const {
		570	return State == NamedModuleImplementation && !getName().contains(':');
		571	}
		572
		573	StringRef getName() const {
		574	assert(isNamedModule() && "Can't get name from a non named module");
		575	return Name;
		576	}
		577
		578	StringRef getPrimaryName() const {
		579	assert(isNamedModule() && "Can't get name from a non named module");
		580	return getName().split(':').first;
		581	}
		582
		583	void reset() {
		584	Name.clear();
		585	State = NotAModuleDecl;
		586	}
		587
		588	private:
		589	ModuleDeclState State;
		590	std::string Name;
		591	};
		592
		593	ModuleDeclSeq ModuleDeclState;
		594
		595	/// Whether the module import expects an identifier next. Otherwise,
		596	/// it expects a '.' or ';'.
		597	bool ModuleImportExpectsIdentifier = false;
		598
		599	/// The identifier and source location of the currently-active
		600	/// \#pragma clang arc_cf_code_audited begin.
		601	std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
		602
		603	/// The source location of the currently-active
		604	/// \#pragma clang assume_nonnull begin.
		605	SourceLocation PragmaAssumeNonNullLoc;
		606
		607	/// Set only for preambles which end with an active
		608	/// \#pragma clang assume_nonnull begin.
		609	///
		610	/// When the preamble is loaded into the main file,
		611	/// `PragmaAssumeNonNullLoc` will be set to this to
		612	/// replay the unterminated assume_nonnull.
		613	SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
		614
		615	/// True if we hit the code-completion point.
		616	bool CodeCompletionReached = false;
		617
		618	/// The code completion token containing the information
		619	/// on the stem that is to be code completed.
		620	IdentifierInfo *CodeCompletionII = nullptr;
		621
		622	/// Range for the code completion token.
		623	SourceRange CodeCompletionTokenRange;
		624
		625	/// The directory that the main file should be considered to occupy,
		626	/// if it does not correspond to a real file (as happens when building a
		627	/// module).
		628	const DirectoryEntry *MainFileDir = nullptr;
		629
		630	/// The number of bytes that we will initially skip when entering the
		631	/// main file, along with a flag that indicates whether skipping this number
		632	/// of bytes will place the lexer at the start of a line.
		633	///
		634	/// This is used when loading a precompiled preamble.
		635	std::pair<int, bool> SkipMainFilePreamble;
		636
		637	/// Whether we hit an error due to reaching max allowed include depth. Allows
		638	/// to avoid hitting the same error over and over again.
		639	bool HasReachedMaxIncludeDepth = false;
		640
		641	/// The number of currently-active calls to Lex.
		642	///
		643	/// Lex is reentrant, and asking for an (end-of-phase-4) token can often
		644	/// require asking for multiple additional tokens. This counter makes it
		645	/// possible for Lex to detect whether it's producing a token for the end
		646	/// of phase 4 of translation or for some other situation.
		647	unsigned LexLevel = 0;
		648
		649	/// The number of (LexLevel 0) preprocessor tokens.
		650	unsigned TokenCount = 0;
		651
		652	/// Preprocess every token regardless of LexLevel.
		653	bool PreprocessToken = false;
		654
		655	/// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
		656	/// warning, or zero for unlimited.
		657	unsigned MaxTokens = 0;
		658	SourceLocation MaxTokensOverrideLoc;
		659
		660	public:
		661	struct PreambleSkipInfo {
		662	SourceLocation HashTokenLoc;
		663	SourceLocation IfTokenLoc;
		664	bool FoundNonSkipPortion;
		665	bool FoundElse;
		666	SourceLocation ElseLoc;
		667
		668	PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
		669	bool FoundNonSkipPortion, bool FoundElse,
		670	SourceLocation ElseLoc)
		671	: HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
		672	FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
		673	ElseLoc(ElseLoc) {}
		674	};
		675
		676	using IncludedFilesSet = llvm::DenseSet<const FileEntry *>;
		677
		678	private:
		679	friend class ASTReader;
		680	friend class MacroArgs;
		681
		682	class PreambleConditionalStackStore {
		683	enum State {
		684	Off = 0,
		685	Recording = 1,
		686	Replaying = 2,
		687	};
		688
		689	public:
		690	PreambleConditionalStackStore() = default;
		691
		692	void startRecording() { ConditionalStackState = Recording; }
		693	void startReplaying() { ConditionalStackState = Replaying; }
		694	bool isRecording() const { return ConditionalStackState == Recording; }
		695	bool isReplaying() const { return ConditionalStackState == Replaying; }
		696
		697	ArrayRef<PPConditionalInfo> getStack() const {
		698	return ConditionalStack;
		699	}
		700
		701	void doneReplaying() {
		702	ConditionalStack.clear();
		703	ConditionalStackState = Off;
		704	}
		705
		706	void setStack(ArrayRef<PPConditionalInfo> s) {
		707	if (!isRecording() && !isReplaying())
		708	return;
		709	ConditionalStack.clear();
		710	ConditionalStack.append(s.begin(), s.end());
		711	}
		712
		713	bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
		714
		715	bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
		716
		717	void clearSkipInfo() { SkipInfo.reset(); }
		718
		719	std::optional<PreambleSkipInfo> SkipInfo;
		720
		721	private:
		722	SmallVector<PPConditionalInfo, 4> ConditionalStack;
		723	State ConditionalStackState = Off;
		724	} PreambleConditionalStack;
		725
		726	/// The current top of the stack that we're lexing from if
		727	/// not expanding a macro and we are lexing directly from source code.
		728	///
		729	/// Only one of CurLexer, or CurTokenLexer will be non-null.
		730	std::unique_ptr<Lexer> CurLexer;
		731
		732	/// The current top of the stack what we're lexing from
		733	/// if not expanding a macro.
		734	///
		735	/// This is an alias for CurLexer.
		736	PreprocessorLexer *CurPPLexer = nullptr;
		737
		738	/// Used to find the current FileEntry, if CurLexer is non-null
		739	/// and if applicable.
		740	///
		741	/// This allows us to implement \#include_next and find directory-specific
		742	/// properties.
		743	ConstSearchDirIterator CurDirLookup = nullptr;
		744
		745	/// The current macro we are expanding, if we are expanding a macro.
		746	///
		747	/// One of CurLexer and CurTokenLexer must be null.
		748	std::unique_ptr<TokenLexer> CurTokenLexer;
		749
		750	/// The kind of lexer we're currently working with.
		751	enum CurLexerKind {
		752	CLK_Lexer,
		753	CLK_TokenLexer,
		754	CLK_CachingLexer,
		755	CLK_DependencyDirectivesLexer,
		756	CLK_LexAfterModuleImport
		757	} CurLexerKind = CLK_Lexer;
		758
		759	/// If the current lexer is for a submodule that is being built, this
		760	/// is that submodule.
		761	Module *CurLexerSubmodule = nullptr;
		762
		763	/// Keeps track of the stack of files currently
		764	/// \#included, and macros currently being expanded from, not counting
		765	/// CurLexer/CurTokenLexer.
		766	struct IncludeStackInfo {
		767	enum CurLexerKind CurLexerKind;
		768	Module *TheSubmodule;
		769	std::unique_ptr<Lexer> TheLexer;
		770	PreprocessorLexer *ThePPLexer;
		771	std::unique_ptr<TokenLexer> TheTokenLexer;
		772	ConstSearchDirIterator TheDirLookup;
		773
		774	// The following constructors are completely useless copies of the default
		775	// versions, only needed to pacify MSVC.
		776	IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
		777	std::unique_ptr<Lexer> &&TheLexer,
		778	PreprocessorLexer *ThePPLexer,
		779	std::unique_ptr<TokenLexer> &&TheTokenLexer,
		780	ConstSearchDirIterator TheDirLookup)
		781	: CurLexerKind(std::move(CurLexerKind)),
		782	TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
		783	ThePPLexer(std::move(ThePPLexer)),
		784	TheTokenLexer(std::move(TheTokenLexer)),
		785	TheDirLookup(std::move(TheDirLookup)) {}
		786	};
		787	std::vector<IncludeStackInfo> IncludeMacroStack;
		788
		789	/// Actions invoked when some preprocessor activity is
		790	/// encountered (e.g. a file is \#included, etc).
		791	std::unique_ptr<PPCallbacks> Callbacks;
		792
		793	struct MacroExpandsInfo {
		794	Token Tok;
		795	MacroDefinition MD;
		796	SourceRange Range;
		797
		798	MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
		799	: Tok(Tok), MD(MD), Range(Range) {}
		800	};
		801	SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
		802
		803	/// Information about a name that has been used to define a module macro.
		804	struct ModuleMacroInfo {
		805	/// The most recent macro directive for this identifier.
		806	MacroDirective *MD;
		807
		808	/// The active module macros for this identifier.
		809	llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
		810
		811	/// The generation number at which we last updated ActiveModuleMacros.
		812	/// \see Preprocessor::VisibleModules.
		813	unsigned ActiveModuleMacrosGeneration = 0;
		814
		815	/// Whether this macro name is ambiguous.
		816	bool IsAmbiguous = false;
		817
		818	/// The module macros that are overridden by this macro.
		819	llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
		820
		821	ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
		822	};
		823
		824	/// The state of a macro for an identifier.
		825	class MacroState {
		826	mutable llvm::PointerUnion<MacroDirective , ModuleMacroInfo > State;
		827
		828	ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
		829	const IdentifierInfo *II) const {
		830	if (II->isOutOfDate())
		831	PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
		832	// FIXME: Find a spare bit on IdentifierInfo and store a
		833	// HasModuleMacros flag.
		834	if (!II->hasMacroDefinition() \|\|
		835	(!PP.getLangOpts().Modules &&
		836	!PP.getLangOpts().ModulesLocalVisibility) \|\|
		837	!PP.CurSubmoduleState->VisibleModules.getGeneration())
		838	return nullptr;
		839
		840	auto Info = State.dyn_cast<ModuleMacroInfo>();
		841	if (!Info) {
		842	Info = new (PP.getPreprocessorAllocator())
		843	ModuleMacroInfo(State.get<MacroDirective *>());
		844	State = Info;
		845	}
		846
		847	if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
		848	Info->ActiveModuleMacrosGeneration)
		849	PP.updateModuleMacroInfo(II, *Info);
		850	return Info;
		851	}
		852
		853	public:
		854	MacroState() : MacroState(nullptr) {}
		855	MacroState(MacroDirective *MD) : State(MD) {}
		856
		857	MacroState(MacroState &&O) noexcept : State(O.State) {
		858	O.State = (MacroDirective *)nullptr;
		859	}
		860
		861	MacroState &operator=(MacroState &&O) noexcept {
		862	auto S = O.State;
		863	O.State = (MacroDirective *)nullptr;
		864	State = S;
		865	return *this;
		866	}
		867
		868	~MacroState() {
		869	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
		870	Info->~ModuleMacroInfo();
		871	}
		872
		873	MacroDirective *getLatest() const {
		874	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
		875	return Info->MD;
		876	return State.get<MacroDirective*>();
		877	}
		878
		879	void setLatest(MacroDirective *MD) {
		880	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
		881	Info->MD = MD;
		882	else
		883	State = MD;
		884	}
		885
		886	bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
		887	auto *Info = getModuleInfo(PP, II);
		888	return Info ? Info->IsAmbiguous : false;
		889	}
		890
		891	ArrayRef<ModuleMacro *>
		892	getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
		893	if (auto *Info = getModuleInfo(PP, II))
		894	return Info->ActiveModuleMacros;
		895	return std::nullopt;
		896	}
		897
		898	MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
		899	SourceManager &SourceMgr) const {
		900	// FIXME: Incorporate module macros into the result of this.
		901	if (auto *Latest = getLatest())
		902	return Latest->findDirectiveAtLoc(Loc, SourceMgr);
		903	return {};
		904	}
		905
		906	void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
		907	if (auto *Info = getModuleInfo(PP, II)) {
		908	Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
		909	Info->ActiveModuleMacros.begin(),
		910	Info->ActiveModuleMacros.end());
		911	Info->ActiveModuleMacros.clear();
		912	Info->IsAmbiguous = false;
		913	}
		914	}
		915
		916	ArrayRef<ModuleMacro*> getOverriddenMacros() const {
		917	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
		918	return Info->OverriddenMacros;
		919	return std::nullopt;
		920	}
		921
		922	void setOverriddenMacros(Preprocessor &PP,
		923	ArrayRef<ModuleMacro *> Overrides) {
		924	auto Info = State.dyn_cast<ModuleMacroInfo>();
		925	if (!Info) {
		926	if (Overrides.empty())
		927	return;
		928	Info = new (PP.getPreprocessorAllocator())
		929	ModuleMacroInfo(State.get<MacroDirective *>());
		930	State = Info;
		931	}
		932	Info->OverriddenMacros.clear();
		933	Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
		934	Overrides.begin(), Overrides.end());
		935	Info->ActiveModuleMacrosGeneration = 0;
		936	}
		937	};
		938
		939	/// For each IdentifierInfo that was associated with a macro, we
		940	/// keep a mapping to the history of all macro definitions and #undefs in
		941	/// the reverse order (the latest one is in the head of the list).
		942	///
		943	/// This mapping lives within the \p CurSubmoduleState.
		944	using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
		945
		946	struct SubmoduleState;
		947
		948	/// Information about a submodule that we're currently building.
		949	struct BuildingSubmoduleInfo {
		950	/// The module that we are building.
		951	Module *M;
		952
		953	/// The location at which the module was included.
		954	SourceLocation ImportLoc;
		955
		956	/// Whether we entered this submodule via a pragma.
		957	bool IsPragma;
		958
		959	/// The previous SubmoduleState.
		960	SubmoduleState *OuterSubmoduleState;
		961
		962	/// The number of pending module macro names when we started building this.
		963	unsigned OuterPendingModuleMacroNames;
		964
		965	BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
		966	SubmoduleState *OuterSubmoduleState,
		967	unsigned OuterPendingModuleMacroNames)
		968	: M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
		969	OuterSubmoduleState(OuterSubmoduleState),
		970	OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
		971	};
		972	SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
		973
		974	/// Information about a submodule's preprocessor state.
		975	struct SubmoduleState {
		976	/// The macros for the submodule.
		977	MacroMap Macros;
		978
		979	/// The set of modules that are visible within the submodule.
		980	VisibleModuleSet VisibleModules;
		981
		982	// FIXME: CounterValue?
		983	// FIXME: PragmaPushMacroInfo?
		984	};
		985	std::map<Module *, SubmoduleState> Submodules;
		986
		987	/// The preprocessor state for preprocessing outside of any submodule.
		988	SubmoduleState NullSubmoduleState;
		989
		990	/// The current submodule state. Will be \p NullSubmoduleState if we're not
		991	/// in a submodule.
		992	SubmoduleState *CurSubmoduleState;
		993
		994	/// The files that have been included.
		995	IncludedFilesSet IncludedFiles;
		996
		997	/// The set of top-level modules that affected preprocessing, but were not
		998	/// imported.
		999	llvm::SmallSetVector<Module *, 2> AffectingClangModules;
		1000
		1001	/// The set of known macros exported from modules.
		1002	llvm::FoldingSet<ModuleMacro> ModuleMacros;
		1003
		1004	/// The names of potential module macros that we've not yet processed.
		1005	llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
		1006
		1007	/// The list of module macros, for each identifier, that are not overridden by
		1008	/// any other module macro.
		1009	llvm::DenseMap<const IdentifierInfo , llvm::TinyPtrVector<ModuleMacro >>
		1010	LeafModuleMacros;
		1011
		1012	/// Macros that we want to warn because they are not used at the end
		1013	/// of the translation unit.
		1014	///
		1015	/// We store just their SourceLocations instead of
		1016	/// something like MacroInfo*. The benefit of this is that when we are
		1017	/// deserializing from PCH, we don't need to deserialize identifier & macros
		1018	/// just so that we can report that they are unused, we just warn using
		1019	/// the SourceLocations of this set (that will be filled by the ASTReader).
		1020	using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
		1021	WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
		1022
		1023	/// This is a pair of an optional message and source location used for pragmas
		1024	/// that annotate macros like pragma clang restrict_expansion and pragma clang
		1025	/// deprecated. This pair stores the optional message and the location of the
		1026	/// annotation pragma for use producing diagnostics and notes.
		1027	using MsgLocationPair = std::pair<std::string, SourceLocation>;
		1028
		1029	struct MacroAnnotationInfo {
		1030	SourceLocation Location;
		1031	std::string Message;
		1032	};
		1033
		1034	struct MacroAnnotations {
		1035	std::optional<MacroAnnotationInfo> DeprecationInfo;
		1036	std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
		1037	std::optional<SourceLocation> FinalAnnotationLoc;
		1038
		1039	static MacroAnnotations makeDeprecation(SourceLocation Loc,
		1040	std::string Msg) {
		1041	return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
		1042	std::nullopt, std::nullopt};
		1043	}
		1044
		1045	static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
		1046	std::string Msg) {
		1047	return MacroAnnotations{
		1048	std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
		1049	}
		1050
		1051	static MacroAnnotations makeFinal(SourceLocation Loc) {
		1052	return MacroAnnotations{std::nullopt, std::nullopt, Loc};
		1053	}
		1054	};
		1055
		1056	/// Warning information for macro annotations.
		1057	llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
		1058
		1059	/// A "freelist" of MacroArg objects that can be
		1060	/// reused for quick allocation.
		1061	MacroArgs *MacroArgCache = nullptr;
		1062
		1063	/// For each IdentifierInfo used in a \#pragma push_macro directive,
		1064	/// we keep a MacroInfo stack used to restore the previous macro value.
		1065	llvm::DenseMap<IdentifierInfo , std::vector<MacroInfo >>
		1066	PragmaPushMacroInfo;
		1067
		1068	// Various statistics we track for performance analysis.
		1069	unsigned NumDirectives = 0;
		1070	unsigned NumDefined = 0;
		1071	unsigned NumUndefined = 0;
		1072	unsigned NumPragma = 0;
		1073	unsigned NumIf = 0;
		1074	unsigned NumElse = 0;
		1075	unsigned NumEndif = 0;
		1076	unsigned NumEnteredSourceFiles = 0;
		1077	unsigned MaxIncludeStackDepth = 0;
		1078	unsigned NumMacroExpanded = 0;
		1079	unsigned NumFnMacroExpanded = 0;
		1080	unsigned NumBuiltinMacroExpanded = 0;
		1081	unsigned NumFastMacroExpanded = 0;
		1082	unsigned NumTokenPaste = 0;
		1083	unsigned NumFastTokenPaste = 0;
		1084	unsigned NumSkipped = 0;
		1085
		1086	/// The predefined macros that preprocessor should use from the
		1087	/// command line etc.
		1088	std::string Predefines;
		1089
		1090	/// The file ID for the preprocessor predefines.
		1091	FileID PredefinesFileID;
		1092
		1093	/// The file ID for the PCH through header.
		1094	FileID PCHThroughHeaderFileID;
		1095
		1096	/// Whether tokens are being skipped until a #pragma hdrstop is seen.
		1097	bool SkippingUntilPragmaHdrStop = false;
		1098
		1099	/// Whether tokens are being skipped until the through header is seen.
		1100	bool SkippingUntilPCHThroughHeader = false;
		1101
		1102	/// \{
		1103	/// Cache of macro expanders to reduce malloc traffic.
		1104	enum { TokenLexerCacheSize = 8 };
		1105	unsigned NumCachedTokenLexers;
		1106	std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
		1107	/// \}
		1108
		1109	/// Keeps macro expanded tokens for TokenLexers.
		1110	//
		1111	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
		1112	/// going to lex in the cache and when it finishes the tokens are removed
		1113	/// from the end of the cache.
		1114	SmallVector<Token, 16> MacroExpandedTokens;
		1115	std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
		1116
		1117	/// A record of the macro definitions and expansions that
		1118	/// occurred during preprocessing.
		1119	///
		1120	/// This is an optional side structure that can be enabled with
		1121	/// \c createPreprocessingRecord() prior to preprocessing.
		1122	PreprocessingRecord *Record = nullptr;
		1123
		1124	/// Cached tokens state.
		1125	using CachedTokensTy = SmallVector<Token, 1>;
		1126
		1127	/// Cached tokens are stored here when we do backtracking or
		1128	/// lookahead. They are "lexed" by the CachingLex() method.
		1129	CachedTokensTy CachedTokens;
		1130
		1131	/// The position of the cached token that CachingLex() should
		1132	/// "lex" next.
		1133	///
		1134	/// If it points beyond the CachedTokens vector, it means that a normal
		1135	/// Lex() should be invoked.
		1136	CachedTokensTy::size_type CachedLexPos = 0;
		1137
		1138	/// Stack of backtrack positions, allowing nested backtracks.
		1139	///
		1140	/// The EnableBacktrackAtThisPos() method pushes a position to
		1141	/// indicate where CachedLexPos should be set when the BackTrack() method is
		1142	/// invoked (at which point the last position is popped).
		1143	std::vector<CachedTokensTy::size_type> BacktrackPositions;
		1144
		1145	/// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
		1146	/// This is used to guard against calling this function recursively.
		1147	///
		1148	/// See comments at the use-site for more context about why it is needed.
		1149	bool SkippingExcludedConditionalBlock = false;
		1150
		1151	/// Keeps track of skipped range mappings that were recorded while skipping
		1152	/// excluded conditional directives. It maps the source buffer pointer at
		1153	/// the beginning of a skipped block, to the number of bytes that should be
		1154	/// skipped.
		1155	llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
		1156
		1157	void updateOutOfDateIdentifier(IdentifierInfo &II) const;
		1158
		1159	public:
		1160	Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
		1161	DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
		1162	HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
		1163	IdentifierInfoLookup *IILookup = nullptr,
		1164	bool OwnsHeaderSearch = false,
		1165	TranslationUnitKind TUKind = TU_Complete);
		1166
		1167	~Preprocessor();
		1168
		1169	/// Initialize the preprocessor using information about the target.
		1170	///
		1171	/// \param Target is owned by the caller and must remain valid for the
		1172	/// lifetime of the preprocessor.
		1173	/// \param AuxTarget is owned by the caller and must remain valid for
		1174	/// the lifetime of the preprocessor.
		1175	void Initialize(const TargetInfo &Target,
		1176	const TargetInfo *AuxTarget = nullptr);
		1177
		1178	/// Initialize the preprocessor to parse a model file
		1179	///
		1180	/// To parse model files the preprocessor of the original source is reused to
		1181	/// preserver the identifier table. However to avoid some duplicate
		1182	/// information in the preprocessor some cleanup is needed before it is used
		1183	/// to parse model files. This method does that cleanup.
		1184	void InitializeForModelFile();
		1185
		1186	/// Cleanup after model file parsing
		1187	void FinalizeForModelFile();
		1188
		1189	/// Retrieve the preprocessor options used to initialize this
		1190	/// preprocessor.
		1191	PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
		1192
		1193	DiagnosticsEngine &getDiagnostics() const { return *Diags; }
		1194	void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
		1195
		1196	const LangOptions &getLangOpts() const { return LangOpts; }
		1197	const TargetInfo &getTargetInfo() const { return *Target; }
		1198	const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
		1199	FileManager &getFileManager() const { return FileMgr; }
		1200	SourceManager &getSourceManager() const { return SourceMgr; }
		1201	HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
		1202
		1203	IdentifierTable &getIdentifierTable() { return Identifiers; }
		1204	const IdentifierTable &getIdentifierTable() const { return Identifiers; }
		1205	SelectorTable &getSelectorTable() { return Selectors; }
		1206	Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
		1207	llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
		1208
		1209	void setExternalSource(ExternalPreprocessorSource *Source) {
		1210	ExternalSource = Source;
		1211	}
		1212
		1213	ExternalPreprocessorSource *getExternalSource() const {
		1214	return ExternalSource;
		1215	}
		1216
		1217	/// Retrieve the module loader associated with this preprocessor.
		1218	ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
		1219
		1220	bool hadModuleLoaderFatalFailure() const {
		1221	return TheModuleLoader.HadFatalFailure;
		1222	}
		1223
		1224	/// Retrieve the number of Directives that have been processed by the
		1225	/// Preprocessor.
		1226	unsigned getNumDirectives() const {
		1227	return NumDirectives;
		1228	}
		1229
		1230	/// True if we are currently preprocessing a #if or #elif directive
		1231	bool isParsingIfOrElifDirective() const {
		1232	return ParsingIfOrElifDirective;
		1233	}
		1234
		1235	/// Control whether the preprocessor retains comments in output.
		1236	void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
		1237	this->KeepComments = KeepComments \| KeepMacroComments;
		1238	this->KeepMacroComments = KeepMacroComments;
		1239	}
		1240
		1241	bool getCommentRetentionState() const { return KeepComments; }
		1242
		1243	void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
		1244	bool getPragmasEnabled() const { return PragmasEnabled; }
		1245
		1246	void SetSuppressIncludeNotFoundError(bool Suppress) {
		1247	SuppressIncludeNotFoundError = Suppress;
		1248	}
		1249
		1250	bool GetSuppressIncludeNotFoundError() {
		1251	return SuppressIncludeNotFoundError;
		1252	}
		1253
		1254	/// Sets whether the preprocessor is responsible for producing output or if
		1255	/// it is producing tokens to be consumed by Parse and Sema.
		1256	void setPreprocessedOutput(bool IsPreprocessedOutput) {
		1257	PreprocessedOutput = IsPreprocessedOutput;
		1258	}
		1259
		1260	/// Returns true if the preprocessor is responsible for generating output,
		1261	/// false if it is producing tokens to be consumed by Parse and Sema.
		1262	bool isPreprocessedOutput() const { return PreprocessedOutput; }
		1263
		1264	/// Return true if we are lexing directly from the specified lexer.
		1265	bool isCurrentLexer(const PreprocessorLexer *L) const {
		1266	return CurPPLexer == L;
		1267	}
		1268
		1269	/// Return the current lexer being lexed from.
		1270	///
		1271	/// Note that this ignores any potentially active macro expansions and _Pragma
		1272	/// expansions going on at the time.
		1273	PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
		1274
		1275	/// Return the current file lexer being lexed from.
		1276	///
		1277	/// Note that this ignores any potentially active macro expansions and _Pragma
		1278	/// expansions going on at the time.
		1279	PreprocessorLexer *getCurrentFileLexer() const;
		1280
		1281	/// Return the submodule owning the file being lexed. This may not be
		1282	/// the current module if we have changed modules since entering the file.
		1283	Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
		1284
		1285	/// Returns the FileID for the preprocessor predefines.
		1286	FileID getPredefinesFileID() const { return PredefinesFileID; }
		1287
		1288	/// \{
		1289	/// Accessors for preprocessor callbacks.
		1290	///
		1291	/// Note that this class takes ownership of any PPCallbacks object given to
		1292	/// it.
		1293	PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
		1294	void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
		1295	if (Callbacks)
		1296	C = std::make_unique<PPChainedCallbacks>(std::move(C),
		1297	std::move(Callbacks));
		1298	Callbacks = std::move(C);
		1299	}
		1300	/// \}
		1301
		1302	/// Get the number of tokens processed so far.
		1303	unsigned getTokenCount() const { return TokenCount; }
		1304
		1305	/// Get the max number of tokens before issuing a -Wmax-tokens warning.
		1306	unsigned getMaxTokens() const { return MaxTokens; }
		1307
		1308	void overrideMaxTokens(unsigned Value, SourceLocation Loc) {
		1309	MaxTokens = Value;
		1310	MaxTokensOverrideLoc = Loc;
		1311	};
		1312
		1313	SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
		1314
		1315	/// Register a function that would be called on each token in the final
		1316	/// expanded token stream.
		1317	/// This also reports annotation tokens produced by the parser.
		1318	void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
		1319	OnToken = std::move(F);
		1320	}
		1321
		1322	void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
		1323
		1324	bool isMacroDefined(StringRef Id) {
		1325	return isMacroDefined(&Identifiers.get(Id));
		1326	}
		1327	bool isMacroDefined(const IdentifierInfo *II) {
		1328	return II->hasMacroDefinition() &&
		1329	(!getLangOpts().Modules \|\| (bool)getMacroDefinition(II));
		1330	}
		1331
		1332	/// Determine whether II is defined as a macro within the module M,
		1333	/// if that is a module that we've already preprocessed. Does not check for
		1334	/// macros imported into M.
		1335	bool isMacroDefinedInLocalModule(const IdentifierInfo II, Module M) {
		1336	if (!II->hasMacroDefinition())
		1337	return false;
		1338	auto I = Submodules.find(M);
		1339	if (I == Submodules.end())
		1340	return false;
		1341	auto J = I->second.Macros.find(II);
		1342	if (J == I->second.Macros.end())
		1343	return false;
		1344	auto *MD = J->second.getLatest();
		1345	return MD && MD->isDefined();
		1346	}
		1347
		1348	MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
		1349	if (!II->hasMacroDefinition())
		1350	return {};
		1351
		1352	MacroState &S = CurSubmoduleState->Macros[II];
		1353	auto *MD = S.getLatest();
		1354	while (MD && isa<VisibilityMacroDirective>(MD))
		1355	MD = MD->getPrevious();
		1356	return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
		1357	S.getActiveModuleMacros(*this, II),
		1358	S.isAmbiguous(*this, II));
		1359	}
		1360
		1361	MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
		1362	SourceLocation Loc) {
		1363	if (!II->hadMacroDefinition())
		1364	return {};
		1365
		1366	MacroState &S = CurSubmoduleState->Macros[II];
		1367	MacroDirective::DefInfo DI;
		1368	if (auto *MD = S.getLatest())
		1369	DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
		1370	// FIXME: Compute the set of active module macros at the specified location.
		1371	return MacroDefinition(DI.getDirective(),
		1372	S.getActiveModuleMacros(*this, II),
		1373	S.isAmbiguous(*this, II));
		1374	}
		1375
		1376	/// Given an identifier, return its latest non-imported MacroDirective
		1377	/// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
		1378	MacroDirective getLocalMacroDirective(const IdentifierInfo II) const {
		1379	if (!II->hasMacroDefinition())
		1380	return nullptr;
		1381
		1382	auto *MD = getLocalMacroDirectiveHistory(II);
		1383	if (!MD \|\| MD->getDefinition().isUndefined())
		1384	return nullptr;
		1385
		1386	return MD;
		1387	}
		1388
		1389	const MacroInfo getMacroInfo(const IdentifierInfo II) const {
		1390	return const_cast<Preprocessor*>(this)->getMacroInfo(II);
		1391	}
		1392
		1393	MacroInfo getMacroInfo(const IdentifierInfo II) {
		1394	if (!II->hasMacroDefinition())
		1395	return nullptr;
		1396	if (auto MD = getMacroDefinition(II))
		1397	return MD.getMacroInfo();
		1398	return nullptr;
		1399	}
		1400
		1401	/// Given an identifier, return the latest non-imported macro
		1402	/// directive for that identifier.
		1403	///
		1404	/// One can iterate over all previous macro directives from the most recent
		1405	/// one.
		1406	MacroDirective getLocalMacroDirectiveHistory(const IdentifierInfo II) const;
		1407
		1408	/// Add a directive to the macro directive history for this identifier.
		1409	void appendMacroDirective(IdentifierInfo II, MacroDirective MD);
		1410	DefMacroDirective appendDefMacroDirective(IdentifierInfo II, MacroInfo *MI,
		1411	SourceLocation Loc) {
		1412	DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
		1413	appendMacroDirective(II, MD);
		1414	return MD;
		1415	}
		1416	DefMacroDirective appendDefMacroDirective(IdentifierInfo II,
		1417	MacroInfo *MI) {
		1418	return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
		1419	}
		1420
		1421	/// Set a MacroDirective that was loaded from a PCH file.
		1422	void setLoadedMacroDirective(IdentifierInfo II, MacroDirective ED,
		1423	MacroDirective *MD);
		1424
		1425	/// Register an exported macro for a module and identifier.
		1426	ModuleMacro addModuleMacro(Module Mod, IdentifierInfo II, MacroInfo Macro,
		1427	ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
		1428	ModuleMacro getModuleMacro(Module Mod, const IdentifierInfo *II);
		1429
		1430	/// Get the list of leaf (non-overridden) module macros for a name.
		1431	ArrayRef<ModuleMacro> getLeafModuleMacros(const IdentifierInfo II) const {
		1432	if (II->isOutOfDate())
		1433	updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
		1434	auto I = LeafModuleMacros.find(II);
		1435	if (I != LeafModuleMacros.end())
		1436	return I->second;
		1437	return std::nullopt;
		1438	}
		1439
		1440	/// Get the list of submodules that we're currently building.
		1441	ArrayRef<BuildingSubmoduleInfo> getBuildingSubmodules() const {
		1442	return BuildingSubmoduleStack;
		1443	}
		1444
		1445	/// \{
		1446	/// Iterators for the macro history table. Currently defined macros have
		1447	/// IdentifierInfo::hasMacroDefinition() set and an empty
		1448	/// MacroInfo::getUndefLoc() at the head of the list.
		1449	using macro_iterator = MacroMap::const_iterator;
		1450
		1451	macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
		1452	macro_iterator macro_end(bool IncludeExternalMacros = true) const;
		1453
		1454	llvm::iterator_range<macro_iterator>
		1455	macros(bool IncludeExternalMacros = true) const {
		1456	macro_iterator begin = macro_begin(IncludeExternalMacros);
		1457	macro_iterator end = macro_end(IncludeExternalMacros);
		1458	return llvm::make_range(begin, end);
		1459	}
		1460
		1461	/// \}
		1462
		1463	/// Mark the given clang module as affecting the current clang module or translation unit.
		1464	void markClangModuleAsAffecting(Module *M) {
		1465	assert(M->isModuleMapModule());
		1466	if (!BuildingSubmoduleStack.empty()) {
		1467	if (M != BuildingSubmoduleStack.back().M)
		1468	BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
		1469	} else {
		1470	AffectingClangModules.insert(M);
		1471	}
		1472	}
		1473
		1474	/// Get the set of top-level clang modules that affected preprocessing, but were not
		1475	/// imported.
		1476	const llvm::SmallSetVector<Module *, 2> &getAffectingClangModules() const {
		1477	return AffectingClangModules;
		1478	}
		1479
		1480	/// Mark the file as included.
		1481	/// Returns true if this is the first time the file was included.
		1482	bool markIncluded(const FileEntry *File) {
		1483	HeaderInfo.getFileInfo(File);
		1484	return IncludedFiles.insert(File).second;
		1485	}
		1486
		1487	/// Return true if this header has already been included.
		1488	bool alreadyIncluded(const FileEntry *File) const {
		1489	return IncludedFiles.count(File);
		1490	}
		1491
		1492	/// Get the set of included files.
		1493	IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
		1494	const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
		1495
		1496	/// Return the name of the macro defined before \p Loc that has
		1497	/// spelling \p Tokens. If there are multiple macros with same spelling,
		1498	/// return the last one defined.
		1499	StringRef getLastMacroWithSpelling(SourceLocation Loc,
		1500	ArrayRef<TokenValue> Tokens) const;
		1501
		1502	/// Get the predefines for this processor.
		1503	/// Used by some third-party tools to inspect and add predefines (see
		1504	/// https://github.com/llvm/llvm-project/issues/57483).
		1505	const std::string &getPredefines() const { return Predefines; }
		1506
		1507	/// Set the predefines for this Preprocessor.
		1508	///
		1509	/// These predefines are automatically injected when parsing the main file.
		1510	void setPredefines(std::string P) { Predefines = std::move(P); }
		1511
		1512	/// Return information about the specified preprocessor
		1513	/// identifier token.
		1514	IdentifierInfo *getIdentifierInfo(StringRef Name) const {
		1515	return &Identifiers.get(Name);
		1516	}
		1517
		1518	/// Add the specified pragma handler to this preprocessor.
		1519	///
		1520	/// If \p Namespace is non-null, then it is a token required to exist on the
		1521	/// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
		1522	void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
		1523	void AddPragmaHandler(PragmaHandler *Handler) {
		1524	AddPragmaHandler(StringRef(), Handler);
		1525	}
		1526
		1527	/// Remove the specific pragma handler from this preprocessor.
		1528	///
		1529	/// If \p Namespace is non-null, then it should be the namespace that
		1530	/// \p Handler was added to. It is an error to remove a handler that
		1531	/// has not been registered.
		1532	void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
		1533	void RemovePragmaHandler(PragmaHandler *Handler) {
		1534	RemovePragmaHandler(StringRef(), Handler);
		1535	}
		1536
		1537	/// Install empty handlers for all pragmas (making them ignored).
		1538	void IgnorePragmas();
		1539
		1540	/// Set empty line handler.
		1541	void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
		1542
		1543	EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
		1544
		1545	/// Add the specified comment handler to the preprocessor.
		1546	void addCommentHandler(CommentHandler *Handler);
		1547
		1548	/// Remove the specified comment handler.
		1549	///
		1550	/// It is an error to remove a handler that has not been registered.
		1551	void removeCommentHandler(CommentHandler *Handler);
		1552
		1553	/// Set the code completion handler to the given object.
		1554	void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
		1555	CodeComplete = &Handler;
		1556	}
		1557
		1558	/// Retrieve the current code-completion handler.
		1559	CodeCompletionHandler *getCodeCompletionHandler() const {
		1560	return CodeComplete;
		1561	}
		1562
		1563	/// Clear out the code completion handler.
		1564	void clearCodeCompletionHandler() {
		1565	CodeComplete = nullptr;
		1566	}
		1567
		1568	/// Hook used by the lexer to invoke the "included file" code
		1569	/// completion point.
		1570	void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
		1571
		1572	/// Hook used by the lexer to invoke the "natural language" code
		1573	/// completion point.
		1574	void CodeCompleteNaturalLanguage();
		1575
		1576	/// Set the code completion token for filtering purposes.
		1577	void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
		1578	CodeCompletionII = Filter;
		1579	}
		1580
		1581	/// Set the code completion token range for detecting replacement range later
		1582	/// on.
		1583	void setCodeCompletionTokenRange(const SourceLocation Start,
		1584	const SourceLocation End) {
		1585	CodeCompletionTokenRange = {Start, End};
		1586	}
		1587	SourceRange getCodeCompletionTokenRange() const {
		1588	return CodeCompletionTokenRange;
		1589	}
		1590
		1591	/// Get the code completion token for filtering purposes.
		1592	StringRef getCodeCompletionFilter() {
		1593	if (CodeCompletionII)
		1594	return CodeCompletionII->getName();
		1595	return {};
		1596	}
		1597
		1598	/// Retrieve the preprocessing record, or NULL if there is no
		1599	/// preprocessing record.
		1600	PreprocessingRecord *getPreprocessingRecord() const { return Record; }
		1601
		1602	/// Create a new preprocessing record, which will keep track of
		1603	/// all macro expansions, macro definitions, etc.
		1604	void createPreprocessingRecord();
		1605
		1606	/// Returns true if the FileEntry is the PCH through header.
		1607	bool isPCHThroughHeader(const FileEntry *FE);
		1608
		1609	/// True if creating a PCH with a through header.
		1610	bool creatingPCHWithThroughHeader();
		1611
		1612	/// True if using a PCH with a through header.
		1613	bool usingPCHWithThroughHeader();
		1614
		1615	/// True if creating a PCH with a #pragma hdrstop.
		1616	bool creatingPCHWithPragmaHdrStop();
		1617
		1618	/// True if using a PCH with a #pragma hdrstop.
		1619	bool usingPCHWithPragmaHdrStop();
		1620
		1621	/// Skip tokens until after the #include of the through header or
		1622	/// until after a #pragma hdrstop.
		1623	void SkipTokensWhileUsingPCH();
		1624
		1625	/// Process directives while skipping until the through header or
		1626	/// #pragma hdrstop is found.
		1627	void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
		1628	SourceLocation HashLoc);
		1629
		1630	/// Enter the specified FileID as the main source file,
		1631	/// which implicitly adds the builtin defines etc.
		1632	void EnterMainSourceFile();
		1633
		1634	/// Inform the preprocessor callbacks that processing is complete.
		1635	void EndSourceFile();
		1636
		1637	/// Add a source file to the top of the include stack and
		1638	/// start lexing tokens from it instead of the current buffer.
		1639	///
		1640	/// Emits a diagnostic, doesn't enter the file, and returns true on error.
		1641	bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir,
		1642	SourceLocation Loc, bool IsFirstIncludeOfFile = true);
		1643
		1644	/// Add a Macro to the top of the include stack and start lexing
		1645	/// tokens from it instead of the current buffer.
		1646	///
		1647	/// \param Args specifies the tokens input to a function-like macro.
		1648	/// \param ILEnd specifies the location of the ')' for a function-like macro
		1649	/// or the identifier for an object-like macro.
		1650	void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
		1651	MacroArgs *Args);
		1652
		1653	private:
		1654	/// Add a "macro" context to the top of the include stack,
		1655	/// which will cause the lexer to start returning the specified tokens.
		1656	///
		1657	/// If \p DisableMacroExpansion is true, tokens lexed from the token stream
		1658	/// will not be subject to further macro expansion. Otherwise, these tokens
		1659	/// will be re-macro-expanded when/if expansion is enabled.
		1660	///
		1661	/// If \p OwnsTokens is false, this method assumes that the specified stream
		1662	/// of tokens has a permanent owner somewhere, so they do not need to be
		1663	/// copied. If it is true, it assumes the array of tokens is allocated with
		1664	/// \c new[] and the Preprocessor will delete[] it.
		1665	///
		1666	/// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
		1667	/// set, see the flag documentation for details.
		1668	void EnterTokenStream(const Token *Toks, unsigned NumToks,
		1669	bool DisableMacroExpansion, bool OwnsTokens,
		1670	bool IsReinject);
		1671
		1672	public:
		1673	void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
		1674	bool DisableMacroExpansion, bool IsReinject) {
		1675	EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
		1676	IsReinject);
		1677	}
		1678
		1679	void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
		1680	bool IsReinject) {
		1681	EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
		1682	IsReinject);
		1683	}
		1684
		1685	/// Pop the current lexer/macro exp off the top of the lexer stack.
		1686	///
		1687	/// This should only be used in situations where the current state of the
		1688	/// top-of-stack lexer is known.
		1689	void RemoveTopOfLexerStack();
		1690
		1691	/// From the point that this method is called, and until
		1692	/// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
		1693	/// keeps track of the lexed tokens so that a subsequent Backtrack() call will
		1694	/// make the Preprocessor re-lex the same tokens.
		1695	///
		1696	/// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
		1697	/// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
		1698	/// be combined with the EnableBacktrackAtThisPos calls in reverse order.
		1699	///
		1700	/// NOTE: DO NOT forget to call either CommitBacktrackedTokens or Backtrack
		1701	/// at some point after EnableBacktrackAtThisPos. If you don't, caching of
		1702	/// tokens will continue indefinitely.
		1703	///
		1704	void EnableBacktrackAtThisPos();
		1705
		1706	/// Disable the last EnableBacktrackAtThisPos call.
		1707	void CommitBacktrackedTokens();
		1708
		1709	/// Make Preprocessor re-lex the tokens that were lexed since
		1710	/// EnableBacktrackAtThisPos() was previously called.
		1711	void Backtrack();
		1712
		1713	/// True if EnableBacktrackAtThisPos() was called and
		1714	/// caching of tokens is on.
		1715	bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
		1716
		1717	/// Lex the next token for this preprocessor.
		1718	void Lex(Token &Result);
		1719
		1720	/// Lex a token, forming a header-name token if possible.
		1721	bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
		1722
		1723	bool LexAfterModuleImport(Token &Result);
		1724	void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
		1725
		1726	void makeModuleVisible(Module *M, SourceLocation Loc);
		1727
		1728	SourceLocation getModuleImportLoc(Module *M) const {
		1729	return CurSubmoduleState->VisibleModules.getImportLoc(M);
		1730	}
		1731
		1732	/// Lex a string literal, which may be the concatenation of multiple
		1733	/// string literals and may even come from macro expansion.
		1734	/// \returns true on success, false if a error diagnostic has been generated.
		1735	bool LexStringLiteral(Token &Result, std::string &String,
		1736	const char *DiagnosticTag, bool AllowMacroExpansion) {
		1737	if (AllowMacroExpansion)
		1738	Lex(Result);
		1739	else
		1740	LexUnexpandedToken(Result);
		1741	return FinishLexStringLiteral(Result, String, DiagnosticTag,
		1742	AllowMacroExpansion);
		1743	}
		1744
		1745	/// Complete the lexing of a string literal where the first token has
		1746	/// already been lexed (see LexStringLiteral).
		1747	bool FinishLexStringLiteral(Token &Result, std::string &String,
		1748	const char *DiagnosticTag,
		1749	bool AllowMacroExpansion);
		1750
		1751	/// Lex a token. If it's a comment, keep lexing until we get
		1752	/// something not a comment.
		1753	///
		1754	/// This is useful in -E -C mode where comments would foul up preprocessor
		1755	/// directive handling.
		1756	void LexNonComment(Token &Result) {
		1757	do
		1758	Lex(Result);
		1759	while (Result.getKind() == tok::comment);
		1760	}
		1761
		1762	/// Just like Lex, but disables macro expansion of identifier tokens.
		1763	void LexUnexpandedToken(Token &Result) {
		1764	// Disable macro expansion.
		1765	bool OldVal = DisableMacroExpansion;
		1766	DisableMacroExpansion = true;
		1767	// Lex the token.
		1768	Lex(Result);
		1769
		1770	// Reenable it.
		1771	DisableMacroExpansion = OldVal;
		1772	}
		1773
		1774	/// Like LexNonComment, but this disables macro expansion of
		1775	/// identifier tokens.
		1776	void LexUnexpandedNonComment(Token &Result) {
		1777	do
		1778	LexUnexpandedToken(Result);
		1779	while (Result.getKind() == tok::comment);
		1780	}
		1781
		1782	/// Parses a simple integer literal to get its numeric value. Floating
		1783	/// point literals and user defined literals are rejected. Used primarily to
		1784	/// handle pragmas that accept integer arguments.
		1785	bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
		1786
		1787	/// Disables macro expansion everywhere except for preprocessor directives.
		1788	void SetMacroExpansionOnlyInDirectives() {
		1789	DisableMacroExpansion = true;
		1790	MacroExpansionInDirectivesOverride = true;
		1791	}
		1792
		1793	/// Peeks ahead N tokens and returns that token without consuming any
		1794	/// tokens.
		1795	///
		1796	/// LookAhead(0) returns the next token that would be returned by Lex(),
		1797	/// LookAhead(1) returns the token after it, etc. This returns normal
		1798	/// tokens after phase 5. As such, it is equivalent to using
		1799	/// 'Lex', not 'LexUnexpandedToken'.
		1800	const Token &LookAhead(unsigned N) {
		1801	assert(LexLevel == 0 && "cannot use lookahead while lexing");
		1802	if (CachedLexPos + N < CachedTokens.size())
		1803	return CachedTokens[CachedLexPos+N];
		1804	else
		1805	return PeekAhead(N+1);
		1806	}
		1807
		1808	/// When backtracking is enabled and tokens are cached,
		1809	/// this allows to revert a specific number of tokens.
		1810	///
		1811	/// Note that the number of tokens being reverted should be up to the last
		1812	/// backtrack position, not more.
		1813	void RevertCachedTokens(unsigned N) {
		1814	assert(isBacktrackEnabled() &&
		1815	"Should only be called when tokens are cached for backtracking");
		1816	assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
		1817	&& "Should revert tokens up to the last backtrack position, not more");
		1818	assert(signed(CachedLexPos) - signed(N) >= 0 &&
		1819	"Corrupted backtrack positions ?");
		1820	CachedLexPos -= N;
		1821	}
		1822
		1823	/// Enters a token in the token stream to be lexed next.
		1824	///
		1825	/// If BackTrack() is called afterwards, the token will remain at the
		1826	/// insertion point.
		1827	/// If \p IsReinject is true, resulting token will have Token::IsReinjected
		1828	/// flag set. See the flag documentation for details.
		1829	void EnterToken(const Token &Tok, bool IsReinject) {
		1830	if (LexLevel) {
		1831	// It's not correct in general to enter caching lex mode while in the
		1832	// middle of a nested lexing action.
		1833	auto TokCopy = std::make_unique<Token[]>(1);
		1834	TokCopy[0] = Tok;
		1835	EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
		1836	} else {
		1837	EnterCachingLexMode();
		1838	assert(IsReinject && "new tokens in the middle of cached stream");
		1839	CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
		1840	}
		1841	}
		1842
		1843	/// We notify the Preprocessor that if it is caching tokens (because
		1844	/// backtrack is enabled) it should replace the most recent cached tokens
		1845	/// with the given annotation token. This function has no effect if
		1846	/// backtracking is not enabled.
		1847	///
		1848	/// Note that the use of this function is just for optimization, so that the
		1849	/// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
		1850	/// invoked.
		1851	void AnnotateCachedTokens(const Token &Tok) {
		1852	assert(Tok.isAnnotation() && "Expected annotation token");
		1853	if (CachedLexPos != 0 && isBacktrackEnabled())
		1854	AnnotatePreviousCachedTokens(Tok);
		1855	}
		1856
		1857	/// Get the location of the last cached token, suitable for setting the end
		1858	/// location of an annotation token.
		1859	SourceLocation getLastCachedTokenLocation() const {
		1860	assert(CachedLexPos != 0);
		1861	return CachedTokens[CachedLexPos-1].getLastLoc();
		1862	}
		1863
		1864	/// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
		1865	/// CachedTokens.
		1866	bool IsPreviousCachedToken(const Token &Tok) const;
		1867
		1868	/// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
		1869	/// in \p NewToks.
		1870	///
		1871	/// Useful when a token needs to be split in smaller ones and CachedTokens
		1872	/// most recent token must to be updated to reflect that.
		1873	void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
		1874
		1875	/// Replace the last token with an annotation token.
		1876	///
		1877	/// Like AnnotateCachedTokens(), this routine replaces an
		1878	/// already-parsed (and resolved) token with an annotation
		1879	/// token. However, this routine only replaces the last token with
		1880	/// the annotation token; it does not affect any other cached
		1881	/// tokens. This function has no effect if backtracking is not
		1882	/// enabled.
		1883	void ReplaceLastTokenWithAnnotation(const Token &Tok) {
		1884	assert(Tok.isAnnotation() && "Expected annotation token");
		1885	if (CachedLexPos != 0 && isBacktrackEnabled())
		1886	CachedTokens[CachedLexPos-1] = Tok;
		1887	}
		1888
		1889	/// Enter an annotation token into the token stream.
		1890	void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
		1891	void *AnnotationVal);
		1892
		1893	/// Determine whether it's possible for a future call to Lex to produce an
		1894	/// annotation token created by a previous call to EnterAnnotationToken.
		1895	bool mightHavePendingAnnotationTokens() {
		1896	return CurLexerKind != CLK_Lexer;
		1897	}
		1898
		1899	/// Update the current token to represent the provided
		1900	/// identifier, in order to cache an action performed by typo correction.
		1901	void TypoCorrectToken(const Token &Tok) {
		1902	assert(Tok.getIdentifierInfo() && "Expected identifier token");
		1903	if (CachedLexPos != 0 && isBacktrackEnabled())
		1904	CachedTokens[CachedLexPos-1] = Tok;
		1905	}
		1906
		1907	/// Recompute the current lexer kind based on the CurLexer/
		1908	/// CurTokenLexer pointers.
		1909	void recomputeCurLexerKind();
		1910
		1911	/// Returns true if incremental processing is enabled
		1912	bool isIncrementalProcessingEnabled() const {
		1913	return getLangOpts().IncrementalExtensions;
		1914	}
		1915
		1916	/// Enables the incremental processing
		1917	void enableIncrementalProcessing(bool value = true) {
		1918	// FIXME: Drop this interface.
		1919	const_cast<LangOptions &>(getLangOpts()).IncrementalExtensions = value;
		1920	}
		1921
		1922	/// Specify the point at which code-completion will be performed.
		1923	///
		1924	/// \param File the file in which code completion should occur. If
		1925	/// this file is included multiple times, code-completion will
		1926	/// perform completion the first time it is included. If NULL, this
		1927	/// function clears out the code-completion point.
		1928	///
		1929	/// \param Line the line at which code completion should occur
		1930	/// (1-based).
		1931	///
		1932	/// \param Column the column at which code completion should occur
		1933	/// (1-based).
		1934	///
		1935	/// \returns true if an error occurred, false otherwise.
		1936	bool SetCodeCompletionPoint(const FileEntry *File,
		1937	unsigned Line, unsigned Column);
		1938
		1939	/// Determine if we are performing code completion.
		1940	bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
		1941
		1942	/// Returns the location of the code-completion point.
		1943	///
		1944	/// Returns an invalid location if code-completion is not enabled or the file
		1945	/// containing the code-completion point has not been lexed yet.
		1946	SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
		1947
		1948	/// Returns the start location of the file of code-completion point.
		1949	///
		1950	/// Returns an invalid location if code-completion is not enabled or the file
		1951	/// containing the code-completion point has not been lexed yet.
		1952	SourceLocation getCodeCompletionFileLoc() const {
		1953	return CodeCompletionFileLoc;
		1954	}
		1955
		1956	/// Returns true if code-completion is enabled and we have hit the
		1957	/// code-completion point.
		1958	bool isCodeCompletionReached() const { return CodeCompletionReached; }
		1959
		1960	/// Note that we hit the code-completion point.
		1961	void setCodeCompletionReached() {
		1962	assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
		1963	CodeCompletionReached = true;
		1964	// Silence any diagnostics that occur after we hit the code-completion.
		1965	getDiagnostics().setSuppressAllDiagnostics(true);
		1966	}
		1967
		1968	/// The location of the currently-active \#pragma clang
		1969	/// arc_cf_code_audited begin.
		1970	///
		1971	/// Returns an invalid location if there is no such pragma active.
		1972	std::pair<IdentifierInfo *, SourceLocation>
		1973	getPragmaARCCFCodeAuditedInfo() const {
		1974	return PragmaARCCFCodeAuditedInfo;
		1975	}
		1976
		1977	/// Set the location of the currently-active \#pragma clang
		1978	/// arc_cf_code_audited begin. An invalid location ends the pragma.
		1979	void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
		1980	SourceLocation Loc) {
		1981	PragmaARCCFCodeAuditedInfo = {Ident, Loc};
		1982	}
		1983
		1984	/// The location of the currently-active \#pragma clang
		1985	/// assume_nonnull begin.
		1986	///
		1987	/// Returns an invalid location if there is no such pragma active.
		1988	SourceLocation getPragmaAssumeNonNullLoc() const {
		1989	return PragmaAssumeNonNullLoc;
		1990	}
		1991
		1992	/// Set the location of the currently-active \#pragma clang
		1993	/// assume_nonnull begin. An invalid location ends the pragma.
		1994	void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
		1995	PragmaAssumeNonNullLoc = Loc;
		1996	}
		1997
		1998	/// Get the location of the recorded unterminated \#pragma clang
		1999	/// assume_nonnull begin in the preamble, if one exists.
		2000	///
		2001	/// Returns an invalid location if the premable did not end with
		2002	/// such a pragma active or if there is no recorded preamble.
		2003	SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const {
		2004	return PreambleRecordedPragmaAssumeNonNullLoc;
		2005	}
		2006
		2007	/// Record the location of the unterminated \#pragma clang
		2008	/// assume_nonnull begin in the preamble.
		2009	void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc) {
		2010	PreambleRecordedPragmaAssumeNonNullLoc = Loc;
		2011	}
		2012
		2013	/// Set the directory in which the main file should be considered
		2014	/// to have been found, if it is not a real file.
		2015	void setMainFileDir(const DirectoryEntry *Dir) {
		2016	MainFileDir = Dir;
		2017	}
		2018
		2019	/// Instruct the preprocessor to skip part of the main source file.
		2020	///
		2021	/// \param Bytes The number of bytes in the preamble to skip.
		2022	///
		2023	/// \param StartOfLine Whether skipping these bytes puts the lexer at the
		2024	/// start of a line.
		2025	void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
		2026	SkipMainFilePreamble.first = Bytes;
		2027	SkipMainFilePreamble.second = StartOfLine;
		2028	}
		2029
		2030	/// Forwarding function for diagnostics. This emits a diagnostic at
		2031	/// the specified Token's location, translating the token's start
		2032	/// position in the current buffer into a SourcePosition object for rendering.
		2033	DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
		2034	return Diags->Report(Loc, DiagID);
		2035	}
		2036
		2037	DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
		2038	return Diags->Report(Tok.getLocation(), DiagID);
		2039	}
		2040
		2041	/// Return the 'spelling' of the token at the given
		2042	/// location; does not go up to the spelling location or down to the
		2043	/// expansion location.
		2044	///
		2045	/// \param buffer A buffer which will be used only if the token requires
		2046	/// "cleaning", e.g. if it contains trigraphs or escaped newlines
		2047	/// \param invalid If non-null, will be set \c true if an error occurs.
		2048	StringRef getSpelling(SourceLocation loc,
		2049	SmallVectorImpl<char> &buffer,
		2050	bool *invalid = nullptr) const {
		2051	return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
		2052	}
		2053
		2054	/// Return the 'spelling' of the Tok token.
		2055	///
		2056	/// The spelling of a token is the characters used to represent the token in
		2057	/// the source file after trigraph expansion and escaped-newline folding. In
		2058	/// particular, this wants to get the true, uncanonicalized, spelling of
		2059	/// things like digraphs, UCNs, etc.
		2060	///
		2061	/// \param Invalid If non-null, will be set \c true if an error occurs.
		2062	std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
		2063	return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
		2064	}
		2065
		2066	/// Get the spelling of a token into a preallocated buffer, instead
		2067	/// of as an std::string.
		2068	///
		2069	/// The caller is required to allocate enough space for the token, which is
		2070	/// guaranteed to be at least Tok.getLength() bytes long. The length of the
		2071	/// actual result is returned.
		2072	///
		2073	/// Note that this method may do two possible things: it may either fill in
		2074	/// the buffer specified with characters, or it may change the input pointer
		2075	/// to point to a constant buffer with the data already in it (avoiding a
		2076	/// copy). The caller is not allowed to modify the returned buffer pointer
		2077	/// if an internal buffer is returned.
		2078	unsigned getSpelling(const Token &Tok, const char *&Buffer,
		2079	bool *Invalid = nullptr) const {
		2080	return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
		2081	}
		2082
		2083	/// Get the spelling of a token into a SmallVector.
		2084	///
		2085	/// Note that the returned StringRef may not point to the
		2086	/// supplied buffer if a copy can be avoided.
		2087	StringRef getSpelling(const Token &Tok,
		2088	SmallVectorImpl<char> &Buffer,
		2089	bool *Invalid = nullptr) const;
		2090
		2091	/// Relex the token at the specified location.
		2092	/// \returns true if there was a failure, false on success.
		2093	bool getRawToken(SourceLocation Loc, Token &Result,
		2094	bool IgnoreWhiteSpace = false) {
		2095	return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
		2096	}
		2097
		2098	/// Given a Token \p Tok that is a numeric constant with length 1,
		2099	/// return the character.
		2100	char
		2101	getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
		2102	bool *Invalid = nullptr) const {
		2103	assert(Tok.is(tok::numeric_constant) &&
		2104	Tok.getLength() == 1 && "Called on unsupported token");
		2105	assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
		2106
		2107	// If the token is carrying a literal data pointer, just use it.
		2108	if (const char *D = Tok.getLiteralData())
		2109	return *D;
		2110
		2111	// Otherwise, fall back on getCharacterData, which is slower, but always
		2112	// works.
		2113	return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
		2114	}
		2115
		2116	/// Retrieve the name of the immediate macro expansion.
		2117	///
		2118	/// This routine starts from a source location, and finds the name of the
		2119	/// macro responsible for its immediate expansion. It looks through any
		2120	/// intervening macro argument expansions to compute this. It returns a
		2121	/// StringRef that refers to the SourceManager-owned buffer of the source
		2122	/// where that macro name is spelled. Thus, the result shouldn't out-live
		2123	/// the SourceManager.
		2124	StringRef getImmediateMacroName(SourceLocation Loc) {
		2125	return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
		2126	}
		2127
		2128	/// Plop the specified string into a scratch buffer and set the
		2129	/// specified token's location and length to it.
		2130	///
		2131	/// If specified, the source location provides a location of the expansion
		2132	/// point of the token.
		2133	void CreateString(StringRef Str, Token &Tok,
		2134	SourceLocation ExpansionLocStart = SourceLocation(),
		2135	SourceLocation ExpansionLocEnd = SourceLocation());
		2136
		2137	/// Split the first Length characters out of the token starting at TokLoc
		2138	/// and return a location pointing to the split token. Re-lexing from the
		2139	/// split token will return the split token rather than the original.
		2140	SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
		2141
		2142	/// Computes the source location just past the end of the
		2143	/// token at this source location.
		2144	///
		2145	/// This routine can be used to produce a source location that
		2146	/// points just past the end of the token referenced by \p Loc, and
		2147	/// is generally used when a diagnostic needs to point just after a
		2148	/// token where it expected something different that it received. If
		2149	/// the returned source location would not be meaningful (e.g., if
		2150	/// it points into a macro), this routine returns an invalid
		2151	/// source location.
		2152	///
		2153	/// \param Offset an offset from the end of the token, where the source
		2154	/// location should refer to. The default offset (0) produces a source
		2155	/// location pointing just past the end of the token; an offset of 1 produces
		2156	/// a source location pointing to the last character in the token, etc.
		2157	SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
		2158	return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
		2159	}
		2160
		2161	/// Returns true if the given MacroID location points at the first
		2162	/// token of the macro expansion.
		2163	///
		2164	/// \param MacroBegin If non-null and function returns true, it is set to
		2165	/// begin location of the macro.
		2166	bool isAtStartOfMacroExpansion(SourceLocation loc,
		2167	SourceLocation *MacroBegin = nullptr) const {
		2168	return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
		2169	MacroBegin);
		2170	}
		2171
		2172	/// Returns true if the given MacroID location points at the last
		2173	/// token of the macro expansion.
		2174	///
		2175	/// \param MacroEnd If non-null and function returns true, it is set to
		2176	/// end location of the macro.
		2177	bool isAtEndOfMacroExpansion(SourceLocation loc,
		2178	SourceLocation *MacroEnd = nullptr) const {
		2179	return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
		2180	}
		2181
		2182	/// Print the token to stderr, used for debugging.
		2183	void DumpToken(const Token &Tok, bool DumpFlags = false) const;
		2184	void DumpLocation(SourceLocation Loc) const;
		2185	void DumpMacro(const MacroInfo &MI) const;
		2186	void dumpMacroInfo(const IdentifierInfo *II);
		2187
		2188	/// Given a location that specifies the start of a
		2189	/// token, return a new location that specifies a character within the token.
		2190	SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
		2191	unsigned Char) const {
		2192	return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
		2193	}
		2194
		2195	/// Increment the counters for the number of token paste operations
		2196	/// performed.
		2197	///
		2198	/// If fast was specified, this is a 'fast paste' case we handled.
		2199	void IncrementPasteCounter(bool isFast) {
		2200	if (isFast)
		2201	++NumFastTokenPaste;
		2202	else
		2203	++NumTokenPaste;
		2204	}
		2205
		2206	void PrintStats();
		2207
		2208	size_t getTotalMemory() const;
		2209
		2210	/// When the macro expander pastes together a comment (/##/) in Microsoft
		2211	/// mode, this method handles updating the current state, returning the
		2212	/// token on the next source line.
		2213	void HandleMicrosoftCommentPaste(Token &Tok);
		2214
		2215	//===--------------------------------------------------------------------===//
		2216	// Preprocessor callback methods. These are invoked by a lexer as various
		2217	// directives and events are found.
		2218
		2219	/// Given a tok::raw_identifier token, look up the
		2220	/// identifier information for the token and install it into the token,
		2221	/// updating the token kind accordingly.
		2222	IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
		2223
		2224	private:
		2225	llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
		2226
		2227	public:
		2228	/// Specifies the reason for poisoning an identifier.
		2229	///
		2230	/// If that identifier is accessed while poisoned, then this reason will be
		2231	/// used instead of the default "poisoned" diagnostic.
		2232	void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
		2233
		2234	/// Display reason for poisoned identifier.
		2235	void HandlePoisonedIdentifier(Token & Identifier);
		2236
		2237	void MaybeHandlePoisonedIdentifier(Token & Identifier) {
		2238	if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
		2239	if(II->isPoisoned()) {
		2240	HandlePoisonedIdentifier(Identifier);
		2241	}
		2242	}
		2243	}
		2244
		2245	private:
		2246	/// Identifiers used for SEH handling in Borland. These are only
		2247	/// allowed in particular circumstances
		2248	// __except block
		2249	IdentifierInfo *Ident__exception_code,
		2250	*Ident___exception_code,
		2251	*Ident_GetExceptionCode;
		2252	// __except filter expression
		2253	IdentifierInfo *Ident__exception_info,
		2254	*Ident___exception_info,
		2255	*Ident_GetExceptionInfo;
		2256	// __finally
		2257	IdentifierInfo *Ident__abnormal_termination,
		2258	*Ident___abnormal_termination,
		2259	*Ident_AbnormalTermination;
		2260
		2261	const char *getCurLexerEndPos();
		2262	void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
		2263
		2264	public:
		2265	void PoisonSEHIdentifiers(bool Poison = true); // Borland
		2266
		2267	/// Callback invoked when the lexer reads an identifier and has
		2268	/// filled in the tokens IdentifierInfo member.
		2269	///
		2270	/// This callback potentially macro expands it or turns it into a named
		2271	/// token (like 'for').
		2272	///
		2273	/// \returns true if we actually computed a token, false if we need to
		2274	/// lex again.
		2275	bool HandleIdentifier(Token &Identifier);
		2276
		2277	/// Callback invoked when the lexer hits the end of the current file.
		2278	///
		2279	/// This either returns the EOF token and returns true, or
		2280	/// pops a level off the include stack and returns false, at which point the
		2281	/// client should call lex again.
		2282	bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
		2283
		2284	/// Callback invoked when the current TokenLexer hits the end of its
		2285	/// token stream.
		2286	bool HandleEndOfTokenLexer(Token &Result);
		2287
		2288	/// Callback invoked when the lexer sees a # token at the start of a
		2289	/// line.
		2290	///
		2291	/// This consumes the directive, modifies the lexer/preprocessor state, and
		2292	/// advances the lexer(s) so that the next token read is the correct one.
		2293	void HandleDirective(Token &Result);
		2294
		2295	/// Ensure that the next token is a tok::eod token.
		2296	///
		2297	/// If not, emit a diagnostic and consume up until the eod.
		2298	/// If \p EnableMacros is true, then we consider macros that expand to zero
		2299	/// tokens as being ok.
		2300	///
		2301	/// \return The location of the end of the directive (the terminating
		2302	/// newline).
		2303	SourceLocation CheckEndOfDirective(const char *DirType,
		2304	bool EnableMacros = false);
		2305
		2306	/// Read and discard all tokens remaining on the current line until
		2307	/// the tok::eod token is found. Returns the range of the skipped tokens.
		2308	SourceRange DiscardUntilEndOfDirective();
		2309
		2310	/// Returns true if the preprocessor has seen a use of
		2311	/// __DATE__ or __TIME__ in the file so far.
		2312	bool SawDateOrTime() const {
		2313	return DATELoc != SourceLocation() \|\| TIMELoc != SourceLocation();
		2314	}
		2315	unsigned getCounterValue() const { return CounterValue; }
		2316	void setCounterValue(unsigned V) { CounterValue = V; }
		2317
		2318	LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const {
		2319	assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
		2320	"FPEvalMethod should be set either from command line or from the "
		2321	"target info");
		2322	return CurrentFPEvalMethod;
		2323	}
		2324
		2325	LangOptions::FPEvalMethodKind getTUFPEvalMethod() const {
		2326	return TUFPEvalMethod;
		2327	}
		2328
		2329	SourceLocation getLastFPEvalPragmaLocation() const {
		2330	return LastFPEvalPragmaLocation;
		2331	}
		2332
		2333	void setCurrentFPEvalMethod(SourceLocation PragmaLoc,
		2334	LangOptions::FPEvalMethodKind Val) {
		2335	assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
		2336	"FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
		2337	// This is the location of the '#pragma float_control" where the
		2338	// execution state is modifed.
		2339	LastFPEvalPragmaLocation = PragmaLoc;
		2340	CurrentFPEvalMethod = Val;
		2341	TUFPEvalMethod = Val;
		2342	}
		2343
		2344	void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val) {
		2345	assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
		2346	"TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
		2347	TUFPEvalMethod = Val;
		2348	}
		2349
		2350	/// Retrieves the module that we're currently building, if any.
		2351	Module *getCurrentModule();
		2352
		2353	/// Retrieves the module whose implementation we're current compiling, if any.
		2354	Module *getCurrentModuleImplementation();
		2355
		2356	/// If we are preprocessing a named module.
		2357	bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
		2358
		2359	/// If we are proprocessing a named interface unit.
		2360	/// Note that a module implementation partition is not considered as an
		2361	/// named interface unit here although it is importable
		2362	/// to ease the parsing.
		2363	bool isInNamedInterfaceUnit() const {
		2364	return ModuleDeclState.isNamedInterface();
		2365	}
		2366
		2367	/// Get the named module name we're preprocessing.
		2368	/// Requires we're preprocessing a named module.
		2369	StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
		2370
		2371	/// If we are implementing an implementation module unit.
		2372	/// Note that the module implementation partition is not considered as an
		2373	/// implementation unit.
		2374	bool isInImplementationUnit() const {
		2375	return ModuleDeclState.isImplementationUnit();
		2376	}
		2377
		2378	/// If we're importing a standard C++20 Named Modules.
		2379	bool isInImportingCXXNamedModules() const {
		2380	// NamedModuleImportPath will be non-empty only if we're importing
		2381	// Standard C++ named modules.
		2382	return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
		2383	!IsAtImport;
		2384	}
		2385
		2386	/// Allocate a new MacroInfo object with the provided SourceLocation.
		2387	MacroInfo *AllocateMacroInfo(SourceLocation L);
		2388
		2389	/// Turn the specified lexer token into a fully checked and spelled
		2390	/// filename, e.g. as an operand of \#include.
		2391	///
		2392	/// The caller is expected to provide a buffer that is large enough to hold
		2393	/// the spelling of the filename, but is also expected to handle the case
		2394	/// when this method decides to use a different buffer.
		2395	///
		2396	/// \returns true if the input filename was in <>'s or false if it was
		2397	/// in ""'s.
		2398	bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
		2399
		2400	/// Given a "foo" or \<foo> reference, look up the indicated file.
		2401	///
		2402	/// Returns std::nullopt on failure. \p isAngled indicates whether the file
		2403	/// reference is for system \#include's or not (i.e. using <> instead of "").
		2404	OptionalFileEntryRef
		2405	LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
		2406	ConstSearchDirIterator FromDir, const FileEntry *FromFile,
		2407	ConstSearchDirIterator CurDir, SmallVectorImpl<char> SearchPath,
		2408	SmallVectorImpl<char> *RelativePath,
		2409	ModuleMap::KnownHeader SuggestedModule, bool IsMapped,
		2410	bool *IsFrameworkFound, bool SkipCache = false,
		2411	bool OpenFile = true, bool CacheFailures = true);
		2412
		2413	/// Return true if we're in the top-level file, not in a \#include.
		2414	bool isInPrimaryFile() const;
		2415
		2416	/// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
		2417	/// followed by EOD. Return true if the token is not a valid on-off-switch.
		2418	bool LexOnOffSwitch(tok::OnOffSwitch &Result);
		2419
		2420	bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
		2421	bool *ShadowFlag = nullptr);
		2422
		2423	void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
		2424	Module *LeaveSubmodule(bool ForPragma);
		2425
		2426	private:
		2427	friend void TokenLexer::ExpandFunctionArguments();
		2428
		2429	void PushIncludeMacroStack() {
		2430	assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
		2431	IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
		2432	std::move(CurLexer), CurPPLexer,
		2433	std::move(CurTokenLexer), CurDirLookup);
		2434	CurPPLexer = nullptr;
		2435	}
		2436
		2437	void PopIncludeMacroStack() {
		2438	CurLexer = std::move(IncludeMacroStack.back().TheLexer);
		2439	CurPPLexer = IncludeMacroStack.back().ThePPLexer;
		2440	CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
		2441	CurDirLookup = IncludeMacroStack.back().TheDirLookup;
		2442	CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
		2443	CurLexerKind = IncludeMacroStack.back().CurLexerKind;
		2444	IncludeMacroStack.pop_back();
		2445	}
		2446
		2447	void PropagateLineStartLeadingSpaceInfo(Token &Result);
		2448
		2449	/// Determine whether we need to create module macros for #defines in the
		2450	/// current context.
		2451	bool needModuleMacros() const;
		2452
		2453	/// Update the set of active module macros and ambiguity flag for a module
		2454	/// macro name.
		2455	void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
		2456
		2457	DefMacroDirective AllocateDefMacroDirective(MacroInfo MI,
		2458	SourceLocation Loc);
		2459	UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
		2460	VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
		2461	bool isPublic);
		2462
		2463	/// Lex and validate a macro name, which occurs after a
		2464	/// \#define or \#undef.
		2465	///
		2466	/// \param MacroNameTok Token that represents the name defined or undefined.
		2467	/// \param IsDefineUndef Kind if preprocessor directive.
		2468	/// \param ShadowFlag Points to flag that is set if macro name shadows
		2469	/// a keyword.
		2470	///
		2471	/// This emits a diagnostic, sets the token kind to eod,
		2472	/// and discards the rest of the macro line if the macro name is invalid.
		2473	void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
		2474	bool *ShadowFlag = nullptr);
		2475
		2476	/// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
		2477	/// entire line) of the macro's tokens and adds them to MacroInfo, and while
		2478	/// doing so performs certain validity checks including (but not limited to):
		2479	/// - # (stringization) is followed by a macro parameter
		2480	/// \param MacroNameTok - Token that represents the macro name
		2481	/// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
		2482	///
		2483	/// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
		2484	/// returns a nullptr if an invalid sequence of tokens is encountered.
		2485	MacroInfo *ReadOptionalMacroParameterListAndBody(
		2486	const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
		2487
		2488	/// The ( starting an argument list of a macro definition has just been read.
		2489	/// Lex the rest of the parameters and the closing ), updating \p MI with
		2490	/// what we learn and saving in \p LastTok the last token read.
		2491	/// Return true if an error occurs parsing the arg list.
		2492	bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
		2493
		2494	/// Provide a suggestion for a typoed directive. If there is no typo, then
		2495	/// just skip suggesting.
		2496	///
		2497	/// \param Tok - Token that represents the directive
		2498	/// \param Directive - String reference for the directive name
		2499	void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
		2500
		2501	/// We just read a \#if or related directive and decided that the
		2502	/// subsequent tokens are in the \#if'd out portion of the
		2503	/// file. Lex the rest of the file, until we see an \#endif. If \p
		2504	/// FoundNonSkipPortion is true, then we have already emitted code for part of
		2505	/// this \#if directive, so \#else/\#elif blocks should never be entered. If
		2506	/// \p FoundElse is false, then \#else directives are ok, if not, then we have
		2507	/// already seen one so a \#else directive is a duplicate. When this returns,
		2508	/// the caller can lex the first valid token.
		2509	void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
		2510	SourceLocation IfTokenLoc,
		2511	bool FoundNonSkipPortion, bool FoundElse,
		2512	SourceLocation ElseLoc = SourceLocation());
		2513
		2514	/// Information about the result for evaluating an expression for a
		2515	/// preprocessor directive.
		2516	struct DirectiveEvalResult {
		2517	/// Whether the expression was evaluated as true or not.
		2518	bool Conditional;
		2519
		2520	/// True if the expression contained identifiers that were undefined.
		2521	bool IncludedUndefinedIds;
		2522
		2523	/// The source range for the expression.
		2524	SourceRange ExprRange;
		2525	};
		2526
		2527	/// Evaluate an integer constant expression that may occur after a
		2528	/// \#if or \#elif directive and return a \p DirectiveEvalResult object.
		2529	///
		2530	/// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
		2531	DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
		2532
		2533	/// Process a '__has_include("path")' expression.
		2534	///
		2535	/// Returns true if successful.
		2536	bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
		2537
		2538	/// Process '__has_include_next("path")' expression.
		2539	///
		2540	/// Returns true if successful.
		2541	bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
		2542
		2543	/// Get the directory and file from which to start \#include_next lookup.
		2544	std::pair<ConstSearchDirIterator, const FileEntry *>
		2545	getIncludeNextStart(const Token &IncludeNextTok) const;
		2546
		2547	/// Install the standard preprocessor pragmas:
		2548	/// \#pragma GCC poison/system_header/dependency and \#pragma once.
		2549	void RegisterBuiltinPragmas();
		2550
		2551	/// Register builtin macros such as __LINE__ with the identifier table.
		2552	void RegisterBuiltinMacros();
		2553
		2554	/// If an identifier token is read that is to be expanded as a macro, handle
		2555	/// it and return the next token as 'Tok'. If we lexed a token, return true;
		2556	/// otherwise the caller should lex again.
		2557	bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
		2558
		2559	/// Cache macro expanded tokens for TokenLexers.
		2560	//
		2561	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
		2562	/// going to lex in the cache and when it finishes the tokens are removed
		2563	/// from the end of the cache.
		2564	Token cacheMacroExpandedTokens(TokenLexer tokLexer,
		2565	ArrayRef<Token> tokens);
		2566
		2567	void removeCachedMacroExpandedTokensOfLastLexer();
		2568
		2569	/// Determine whether the next preprocessor token to be
		2570	/// lexed is a '('. If so, consume the token and return true, if not, this
		2571	/// method should have no observable side-effect on the lexed tokens.
		2572	bool isNextPPTokenLParen();
		2573
		2574	/// After reading "MACRO(", this method is invoked to read all of the formal
		2575	/// arguments specified for the macro invocation. Returns null on error.
		2576	MacroArgs ReadMacroCallArgumentList(Token &MacroName, MacroInfo MI,
		2577	SourceLocation &MacroEnd);
		2578
		2579	/// If an identifier token is read that is to be expanded
		2580	/// as a builtin macro, handle it and return the next token as 'Tok'.
		2581	void ExpandBuiltinMacro(Token &Tok);
		2582
		2583	/// Read a \c _Pragma directive, slice it up, process it, then
		2584	/// return the first token after the directive.
		2585	/// This assumes that the \c _Pragma token has just been read into \p Tok.
		2586	void Handle_Pragma(Token &Tok);
		2587
		2588	/// Like Handle_Pragma except the pragma text is not enclosed within
		2589	/// a string literal.
		2590	void HandleMicrosoft__pragma(Token &Tok);
		2591
		2592	/// Add a lexer to the top of the include stack and
		2593	/// start lexing tokens from it instead of the current buffer.
		2594	void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
		2595
		2596	/// Set the FileID for the preprocessor predefines.
		2597	void setPredefinesFileID(FileID FID) {
		2598	assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
		2599	PredefinesFileID = FID;
		2600	}
		2601
		2602	/// Set the FileID for the PCH through header.
		2603	void setPCHThroughHeaderFileID(FileID FID);
		2604
		2605	/// Returns true if we are lexing from a file and not a
		2606	/// pragma or a macro.
		2607	static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
		2608	return L ? !L->isPragmaLexer() : P != nullptr;
		2609	}
		2610
		2611	static bool IsFileLexer(const IncludeStackInfo& I) {
		2612	return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
		2613	}
		2614
		2615	bool IsFileLexer() const {
		2616	return IsFileLexer(CurLexer.get(), CurPPLexer);
		2617	}
		2618
		2619	//===--------------------------------------------------------------------===//
		2620	// Caching stuff.
		2621	void CachingLex(Token &Result);
		2622
		2623	bool InCachingLexMode() const {
		2624	// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
		2625	// that we are past EOF, not that we are in CachingLex mode.
		2626	return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
		2627	}
		2628
		2629	void EnterCachingLexMode();
		2630	void EnterCachingLexModeUnchecked();
		2631
		2632	void ExitCachingLexMode() {
		2633	if (InCachingLexMode())
		2634	RemoveTopOfLexerStack();
		2635	}
		2636
		2637	const Token &PeekAhead(unsigned N);
		2638	void AnnotatePreviousCachedTokens(const Token &Tok);
		2639
		2640	//===--------------------------------------------------------------------===//
		2641	/// Handle*Directive - implement the various preprocessor directives. These
		2642	/// should side-effect the current preprocessor object so that the next call
		2643	/// to Lex() will return the appropriate token next.
		2644	void HandleLineDirective();
		2645	void HandleDigitDirective(Token &Tok);
		2646	void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
		2647	void HandleIdentSCCSDirective(Token &Tok);
		2648	void HandleMacroPublicDirective(Token &Tok);
		2649	void HandleMacroPrivateDirective();
		2650
		2651	/// An additional notification that can be produced by a header inclusion or
		2652	/// import to tell the parser what happened.
		2653	struct ImportAction {
		2654	enum ActionKind {
		2655	None,
		2656	ModuleBegin,
		2657	ModuleImport,
		2658	HeaderUnitImport,
		2659	SkippedModuleImport,
		2660	Failure,
		2661	} Kind;
		2662	Module *ModuleForHeader = nullptr;
		2663
		2664	ImportAction(ActionKind AK, Module *Mod = nullptr)
		2665	: Kind(AK), ModuleForHeader(Mod) {
		2666	assert((AK == None \|\| Mod \|\| AK == Failure) &&
		2667	"no module for module action");
		2668	}
		2669	};
		2670
		2671	OptionalFileEntryRef LookupHeaderIncludeOrImport(
		2672	ConstSearchDirIterator *CurDir, StringRef &Filename,
		2673	SourceLocation FilenameLoc, CharSourceRange FilenameRange,
		2674	const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
		2675	bool &IsMapped, ConstSearchDirIterator LookupFrom,
		2676	const FileEntry *LookupFromFile, StringRef &LookupFilename,
		2677	SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
		2678	ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
		2679
		2680	// File inclusion.
		2681	void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
		2682	ConstSearchDirIterator LookupFrom = nullptr,
		2683	const FileEntry *LookupFromFile = nullptr);
		2684	ImportAction
		2685	HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
		2686	Token &FilenameTok, SourceLocation EndLoc,
		2687	ConstSearchDirIterator LookupFrom = nullptr,
		2688	const FileEntry *LookupFromFile = nullptr);
		2689	void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
		2690	void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
		2691	void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
		2692	void HandleMicrosoftImportDirective(Token &Tok);
		2693
		2694	public:
		2695	/// Check that the given module is available, producing a diagnostic if not.
		2696	/// \return \c true if the check failed (because the module is not available).
		2697	/// \c false if the module appears to be usable.
		2698	static bool checkModuleIsAvailable(const LangOptions &LangOpts,
		2699	const TargetInfo &TargetInfo,
		2700	DiagnosticsEngine &Diags, Module *M);
		2701
		2702	// Module inclusion testing.
		2703	/// Find the module that owns the source or header file that
		2704	/// \p Loc points to. If the location is in a file that was included
		2705	/// into a module, or is outside any module, returns nullptr.
		2706	Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
		2707
		2708	/// We want to produce a diagnostic at location IncLoc concerning an
		2709	/// unreachable effect at location MLoc (eg, where a desired entity was
		2710	/// declared or defined). Determine whether the right way to make MLoc
		2711	/// reachable is by #include, and if so, what header should be included.
		2712	///
		2713	/// This is not necessarily fast, and might load unexpected module maps, so
		2714	/// should only be called by code that intends to produce an error.
		2715	///
		2716	/// \param IncLoc The location at which the missing effect was detected.
		2717	/// \param MLoc A location within an unimported module at which the desired
		2718	/// effect occurred.
		2719	/// \return A file that can be #included to provide the desired effect. Null
		2720	/// if no such file could be determined or if a #include is not
		2721	/// appropriate (eg, if a module should be imported instead).
		2722	const FileEntry *getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
		2723	SourceLocation MLoc);
		2724
		2725	bool isRecordingPreamble() const {
		2726	return PreambleConditionalStack.isRecording();
		2727	}
		2728
		2729	bool hasRecordedPreamble() const {
		2730	return PreambleConditionalStack.hasRecordedPreamble();
		2731	}
		2732
		2733	ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
		2734	return PreambleConditionalStack.getStack();
		2735	}
		2736
		2737	void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
		2738	PreambleConditionalStack.setStack(s);
		2739	}
		2740
		2741	void setReplayablePreambleConditionalStack(
		2742	ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
		2743	PreambleConditionalStack.startReplaying();
		2744	PreambleConditionalStack.setStack(s);
		2745	PreambleConditionalStack.SkipInfo = SkipInfo;
		2746	}
		2747
		2748	std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
		2749	return PreambleConditionalStack.SkipInfo;
		2750	}
		2751
		2752	private:
		2753	/// After processing predefined file, initialize the conditional stack from
		2754	/// the preamble.
		2755	void replayPreambleConditionalStack();
		2756
		2757	// Macro handling.
		2758	void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
		2759	void HandleUndefDirective();
		2760
		2761	// Conditional Inclusion.
		2762	void HandleIfdefDirective(Token &Result, const Token &HashToken,
		2763	bool isIfndef, bool ReadAnyTokensBeforeDirective);
		2764	void HandleIfDirective(Token &IfToken, const Token &HashToken,
		2765	bool ReadAnyTokensBeforeDirective);
		2766	void HandleEndifDirective(Token &EndifToken);
		2767	void HandleElseDirective(Token &Result, const Token &HashToken);
		2768	void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
		2769	tok::PPKeywordKind Kind);
		2770
		2771	// Pragmas.
		2772	void HandlePragmaDirective(PragmaIntroducer Introducer);
		2773
		2774	public:
		2775	void HandlePragmaOnce(Token &OnceTok);
		2776	void HandlePragmaMark(Token &MarkTok);
		2777	void HandlePragmaPoison();
		2778	void HandlePragmaSystemHeader(Token &SysHeaderTok);
		2779	void HandlePragmaDependency(Token &DependencyTok);
		2780	void HandlePragmaPushMacro(Token &Tok);
		2781	void HandlePragmaPopMacro(Token &Tok);
		2782	void HandlePragmaIncludeAlias(Token &Tok);
		2783	void HandlePragmaModuleBuild(Token &Tok);
		2784	void HandlePragmaHdrstop(Token &Tok);
		2785	IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
		2786
		2787	// Return true and store the first token only if any CommentHandler
		2788	// has inserted some tokens and getCommentRetentionState() is false.
		2789	bool HandleComment(Token &result, SourceRange Comment);
		2790
		2791	/// A macro is used, update information about macros that need unused
		2792	/// warnings.
		2793	void markMacroAsUsed(MacroInfo *MI);
		2794
		2795	void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
		2796	SourceLocation AnnotationLoc) {
		2797	auto Annotations = AnnotationInfos.find(II);
		2798	if (Annotations == AnnotationInfos.end())
		2799	AnnotationInfos.insert(std::make_pair(
		2800	II,
		2801	MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
		2802	else
		2803	Annotations->second.DeprecationInfo =
		2804	MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
		2805	}
		2806
		2807	void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
		2808	SourceLocation AnnotationLoc) {
		2809	auto Annotations = AnnotationInfos.find(II);
		2810	if (Annotations == AnnotationInfos.end())
		2811	AnnotationInfos.insert(
		2812	std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
		2813	AnnotationLoc, std::move(Msg))));
		2814	else
		2815	Annotations->second.RestrictExpansionInfo =
		2816	MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
		2817	}
		2818
		2819	void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
		2820	auto Annotations = AnnotationInfos.find(II);
		2821	if (Annotations == AnnotationInfos.end())
		2822	AnnotationInfos.insert(
		2823	std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
		2824	else
		2825	Annotations->second.FinalAnnotationLoc = AnnotationLoc;
		2826	}
		2827
		2828	const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
		2829	return AnnotationInfos.find(II)->second;
		2830	}
		2831
		2832	void emitMacroExpansionWarnings(const Token &Identifier) const {
		2833	if (Identifier.getIdentifierInfo()->isDeprecatedMacro())
		2834	emitMacroDeprecationWarning(Identifier);
		2835
		2836	if (Identifier.getIdentifierInfo()->isRestrictExpansion() &&
		2837	!SourceMgr.isInMainFile(Identifier.getLocation()))
		2838	emitRestrictExpansionWarning(Identifier);
		2839	}
		2840
		2841	static void processPathForFileMacro(SmallVectorImpl<char> &Path,
		2842	const LangOptions &LangOpts,
		2843	const TargetInfo &TI);
		2844
		2845	private:
		2846	void emitMacroDeprecationWarning(const Token &Identifier) const;
		2847	void emitRestrictExpansionWarning(const Token &Identifier) const;
		2848	void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
		2849	};
		2850
		2851	/// Abstract base class that describes a handler that will receive
		2852	/// source ranges for each of the comments encountered in the source file.
		2853	class CommentHandler {
		2854	public:
		2855	virtual ~CommentHandler();
		2856
		2857	// The handler shall return true if it has pushed any tokens
		2858	// to be read using e.g. EnterToken or EnterTokenStream.
		2859	virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
		2860	};
		2861
		2862	/// Abstract base class that describes a handler that will receive
		2863	/// source ranges for empty lines encountered in the source file.
		2864	class EmptylineHandler {
		2865	public:
		2866	virtual ~EmptylineHandler();
		2867
		2868	// The handler handles empty lines.
		2869	virtual void HandleEmptyline(SourceRange Range) = 0;
		2870	};
		2871
		2872	/// Registry of pragma handlers added by plugins
		2873	using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
		2874
		2875	} // namespace clang
		2876
		2877	#endif // LLVM_CLANG_LEX_PREPROCESSOR_H

Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

QNX 8.QNX8 LLVM/Clang compiler suite//llvm-build/x86_64/include/clang/Lex/Preprocessor.h – Rev 14