From 91dd967c66a5478b9b2b39b7331c185455909b06 Mon Sep 17 00:00:00 2001 From: Alisdair Meredith Date: Sat, 9 Nov 2024 11:28:51 -0500 Subject: [PATCH] [lex.pptoken] Separate out placeholder keywords The _preprocessing-token_ term is defined as one of another set of grammar productions, with the three placeholder tokens standing out as specific single tokens. Moving the placeholder tokens into their own production simplifies both the _preprocessing-token_ and _keyword_ grammar productions, plus it guarantees a consistency for the set of placeholder between both places. IT also makes it easier for [cpp] to directly cross-reference where needed. Pedantically, the term placeholder token itself is problematic as tokens are a phase 7 concern, where they become keywords. Within the earlier phases where these placeholders are used the correct term is a preprocessing token, so strictly these should be placeholder preprocessing tokens, which is a mouthful. So renaming the term to placeholder keyword, which is hopefully less of a concern as there are no keywords in phase 3--6 to confuse with phase 7 keywords. Finally, clean up the informative list in text of the various kinds of preprocessing tokens, rendering as a list in the source (one kind per line) and adding a cross-reference to each such item. --- source/lex.tex | 62 +++++++++++++++++++++++++---------------- source/preprocessor.tex | 4 +-- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/source/lex.tex b/source/lex.tex index 7a0f88edee..e595c44f68 100644 --- a/source/lex.tex +++ b/source/lex.tex @@ -498,12 +498,10 @@ \indextext{token!preprocessing|(}% \begin{bnf} \nontermdef{preprocessing-token}\br + keyword-placeholder\br header-name\br - import-keyword\br - module-keyword\br - export-keyword\br - identifier\br pp-number\br + identifier\br character-literal\br user-defined-character-literal\br string-literal\br @@ -518,12 +516,15 @@ In this document, glyphs are used to identify elements of the basic character set\iref{lex.charset}. -The categories of preprocessing token are: header names, -placeholder tokens produced by preprocessing \tcode{import} and \tcode{module} directives -(\grammarterm{import-keyword}, \grammarterm{module-keyword}, and \grammarterm{export-keyword}), -identifiers, preprocessing numbers, character literals (including user-defined character -literals), string literals (including user-defined string literals), preprocessing -operators and punctuators, and single non-whitespace characters that do not lexically +The categories of preprocessing token are: +keyword plaeholders\iref{lex.placeholder}, +header names\iref{lex.header}, +preprocessing numbers\iref{lex.ppnumber}, +identifiers\iref{lex.name}, +character literals\iref{lex.ccon} including user-defined character literals, +string literals\iref{lex.string} including user-defined string literals, +preprocessing operators and punctuators\iref{lex.operators}, and +single non-whitespace characters that do not lexically match the other preprocessing token categories. If a \unicode{0027}{apostrophe} or a \unicode{0022}{quotation mark} character matches the last category, the program is ill-formed. @@ -551,17 +552,6 @@ shall have the lexical form of a keyword, an identifier, a literal, or an operator or punctuator. -\pnum -The \grammarterm{import-keyword} is produced -by processing an \keyword{import} directive\iref{cpp.import}, -the \grammarterm{module-keyword} is produced -by preprocessing a \keyword{module} directive\iref{cpp.module}, and -the \grammarterm{export-keyword} is produced -by preprocessing either of the previous two directives. -\begin{note} -None has any observable spelling. -\end{note} - \pnum If the input stream has been parsed into preprocessing tokens up to a given character: @@ -631,6 +621,32 @@ \end{example} \indextext{token!preprocessing|)} +\rSec1[lex.placeholder]{Keyword placeholders} +\indextext{placeholder!keyword|(}% + +\begin{bnf} +\nontermdef{keyword-placeholder}\br + export-keyword\br + import-keyword\br + module-keyword +\end{bnf} + +\pnum +Keyword placeholders are produced by preprocessing \keyword{import} +and \keyword{module} directives. +The \grammarterm{import-keyword} is produced +by preprocessing an \keyword{import} directive\iref{cpp.import}, +the \grammarterm{module-keyword} is produced +by preprocessing a \keyword{module} directive\iref{cpp.module}, and +the \grammarterm{export-keyword} is produced +by preprocessing either of the previous two directives. + +\pnum +\begin{note} +Keyword placeholders do not have any observable spelling. +\end{note} +\indextext{placeholder!keyword|(}% + \rSec1[lex.header]{Header names} \indextext{header!name|(}% @@ -964,9 +980,7 @@ \begin{bnf} \nontermdef{keyword}\br \textnormal{any identifier listed in \tref{lex.key}}\br - \grammarterm{import-keyword}\br - \grammarterm{module-keyword}\br - \grammarterm{export-keyword} + \grammarterm{keyword-placeholder} \end{bnf} \pnum diff --git a/source/preprocessor.tex b/source/preprocessor.tex index 64a6a72ccc..76feadf3ca 100644 --- a/source/preprocessor.tex +++ b/source/preprocessor.tex @@ -1233,7 +1233,7 @@ \pnum The \tcode{module} and \tcode{export} (if it exists) preprocessing tokens are replaced by the \grammarterm{module-keyword} and -\grammarterm{export-keyword} preprocessing tokens respectively. +\grammarterm{export-keyword} keyword placeholders respectively\iref{lex.placeholder}. \begin{note} This makes the line no longer a directive so it is not removed at the end of phase 4. @@ -1291,7 +1291,7 @@ In all three forms of \grammarterm{pp-import}, the \tcode{import} and \tcode{export} (if it exists) preprocessing tokens are replaced by the \grammarterm{import-keyword} and -\grammarterm{export-keyword} preprocessing tokens respectively. +\grammarterm{export-keyword} keyword placeholders respectively\iref{lex.placeholder}. \begin{note} This makes the line no longer a directive so it is not removed at the end of phase 4.