[flang] Don't perform macro replacement on exponents #136176
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
flang
|
@llvm/pr-subscribers-flang-parser Author: Peter Klausler (klausler) ChangesSee new test. I inadvertently broke this behavior with a recent fix for another problem, because the effects of the overloaded TokenSequence::Put() member function on token merging were confusing. Rename and document the various overloads. Full diff: https://github.com/llvm/llvm-project/pull/136176.diff 5 Files Affected:
diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
index 047c0bed00762..a3a66c575a7d4 100644
--- a/flang/include/flang/Parser/token-sequence.h
+++ b/flang/include/flang/Parser/token-sequence.h
@@ -35,10 +35,10 @@ class Prescanner;
class TokenSequence {
public:
TokenSequence() {}
- TokenSequence(const TokenSequence &that) { Put(that); }
+ TokenSequence(const TokenSequence &that) { CopyAll(that); }
TokenSequence(
const TokenSequence &that, std::size_t at, std::size_t count = 1) {
- Put(that, at, count);
+ AppendRange(that, at, count);
}
TokenSequence(TokenSequence &&that)
: start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@@ -48,7 +48,7 @@ class TokenSequence {
TokenSequence &operator=(const TokenSequence &that) {
clear();
- Put(that);
+ CopyAll(that);
return *this;
}
TokenSequence &operator=(TokenSequence &&that);
@@ -100,14 +100,27 @@ class TokenSequence {
start_.pop_back();
}
- void Put(const TokenSequence &);
- void Put(const TokenSequence &, ProvenanceRange);
- void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+ // These append characters with provenance and then close the token.
+ // When the last token of this sequence remains open beforehand,
+ // the new characters are appended to it.
void Put(const char *, std::size_t, Provenance);
void Put(const CharBlock &, Provenance);
void Put(const std::string &, Provenance);
void Put(llvm::raw_string_ostream &, Provenance);
+ // Closes current token (if open), than appends full copy of another sequence.
+ // Appends a full copy of another sequence. When the last token of this
+ // sequence remains open beforehand, it is closed before the new text
+ // is appended.
+ void CopyAll(const TokenSequence &);
+ // Copies a range of tokens from another sequence. If the last token of
+ // this sequence remains open, the first token of the copied range will be
+ // appended to it.
+ void AppendRange(
+ const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+ // Copies tokens (via Put above) with new provenance.
+ void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
+
Provenance GetCharProvenance(std::size_t) const;
Provenance GetTokenProvenance(
std::size_t token, std::size_t offset = 0) const;
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 7e6a1c2ca6977..a47f9c32ad27c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
continue;
}
}
- result.Put(token, firstToken + j, 1);
+ result.AppendRange(token, firstToken + j, 1);
}
return result;
}
@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
}
} else if (pasting && text.TokenAt(j).IsBlank()) {
} else {
- result.Put(text, j, 1);
+ result.AppendRange(text, j, 1);
pasting = false;
}
}
@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
CHECK(resultSize > 0 &&
result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
result.pop_back();
- result.Put(Stringify(args[index], prescanner.allSources()));
+ result.CopyAll(Stringify(args[index], prescanner.allSources()));
} else {
const TokenSequence *arg{&args[index]};
std::optional<TokenSequence> replaced;
@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
- result.Put(DEREF(arg));
+ result.CopyAll(DEREF(arg));
}
} else if (bytes == 11 && isVariadic_ &&
token.ToString() == "__VA_ARGS__") {
@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
if (k > argumentCount()) {
result.Put(","s, commaProvenance);
}
- result.Put(args[k]);
+ result.CopyAll(args[k]);
}
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
- result.Put(replacement_, j);
+ result.AppendRange(replacement_, j);
}
}
return TokenPasting(std::move(result));
@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
inIfExpression](std::size_t after, const TokenSequence &replacement,
std::size_t pFLMOffset) {
if (after < input.SizeInTokens()) {
- result.Put(replacement, 0, pFLMOffset);
+ result.AppendRange(replacement, 0, pFLMOffset);
TokenSequence suffix;
- suffix.Put(
+ suffix.AppendRange(
replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
- suffix.Put(input, after, input.SizeInTokens() - after);
+ suffix.AppendRange(input, after, input.SizeInTokens() - after);
auto further{ReplaceMacros(
suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
// still not closed
**partialFunctionLikeMacro += result.SizeInTokens();
}
- result.Put(further);
+ result.CopyAll(further);
return true;
} else {
if (partialFunctionLikeMacro) {
@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
for (; j < tokens; ++j) {
CharBlock token{input.TokenAt(j)};
if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
// Process identifier in replacement text.
@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
}
if (it == definitions_.end()) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
Definition *def{&it->second};
if (def->isDisabled()) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
if (!def->isFunctionLike()) {
@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetTokenProvenanceRange(j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
- result.Put(replaced, newRange);
+ result.CopyWithProvenance(replaced, newRange);
}
} else {
// Possible function-like macro call. Skip spaces and newlines to see
@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
if (!leftParen) {
if (partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
- result.Put(input, j, tokens - j);
+ result.AppendRange(input, j, tokens - j);
return result;
} else {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
}
@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
if (k >= tokens && partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
- result.Put(input, j, tokens - j);
+ result.AppendRange(input, j, tokens - j);
return result;
} else if (k >= tokens || argStart.size() < def->argumentCount() ||
(argStart.size() > def->argumentCount() && !def->isVariadic())) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
std::vector<TokenSequence> args;
@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
- result.Put(replaced, newRange);
+ result.CopyWithProvenance(replaced, newRange);
}
j = k; // advance to the terminal ')'
}
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index b2b3d7fcfe786..c1254a9e42954 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
EmitCharAndAdvance(possible, *at_);
}
possible.CloseToken();
- tokens.CloseToken();
- tokens.Put(possible);
+ tokens.AppendRange(possible, 0); // appends to current token
return true;
}
// Not an exponent; backtrack
@@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
!preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
// "_foo" is not defined, but "foo" is
- tokens.Put(separate); // '_' "foo"
+ tokens.CopyAll(separate); // '_' "foo"
} else {
- tokens.Put(withUnderscore); // "_foo"
+ tokens.CopyAll(withUnderscore); // "_foo"
}
return true;
}
@@ -1011,7 +1010,7 @@ void Prescanner::QuotedCharacterLiteral(
ppTokens.Put(id, GetProvenance(idStart));
if (auto replaced{
preprocessor_.MacroReplacement(ppTokens, *this)}) {
- tokens.Put(*replaced);
+ tokens.CopyAll(*replaced);
at_ = &idStart[idLen - 1];
NextLine();
continue; // try again on the next line
@@ -1812,7 +1811,7 @@ bool Prescanner::CompilerDirectiveContinuation(
}
if (ok) {
tokens.pop_back(); // delete original '&'
- tokens.Put(followingTokens, startAt, following - startAt);
+ tokens.AppendRange(followingTokens, startAt, following - startAt);
tokens.RemoveRedundantBlanks();
} else {
nextLine_ = origNextLine;
@@ -1842,7 +1841,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
}
followingTokens.RemoveRedundantBlanks();
tokens.pop_back(); // delete original '&'
- tokens.Put(followingTokens);
+ tokens.CopyAll(followingTokens);
return true;
}
}
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index c0655f608b97d..aee76938550f5 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
return false;
}
-void TokenSequence::Put(const TokenSequence &that) {
+void TokenSequence::CopyAll(const TokenSequence &that) {
if (nextStart_ < char_.size()) {
start_.push_back(nextStart_);
}
@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
provenances_.Put(that.provenances_);
}
-void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
+void TokenSequence::CopyWithProvenance(
+ const TokenSequence &that, ProvenanceRange range) {
std::size_t offset{0};
std::size_t tokens{that.SizeInTokens()};
for (std::size_t j{0}; j < tokens; ++j) {
@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
CHECK(offset == range.size());
}
-void TokenSequence::Put(
+void TokenSequence::AppendRange(
const TokenSequence &that, std::size_t at, std::size_t tokens) {
ProvenanceRange provenance;
std::size_t offset{0};
@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
TokenSequence result;
for (std::size_t j{0}; j < tokens; ++j) {
if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
- result.Put(*this, j);
+ result.AppendRange(*this, j);
}
}
swap(result);
@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
for (std::size_t j{0}; j < tokens; ++j) {
bool isBlank{TokenAt(j).IsBlank()};
if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
- result.Put(*this, j);
+ result.AppendRange(*this, j);
}
lastWasBlank = isBlank;
}
@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
} else {
TokenSequence result;
if (j > 0) {
- result.Put(*this, 0, j - 1);
+ result.AppendRange(*this, 0, j - 1);
}
swap(result);
return *this;
diff --git a/flang/test/Preprocessing/exponent-bug.F90 b/flang/test/Preprocessing/exponent-bug.F90
new file mode 100644
index 0000000000000..903b41bb846a4
--- /dev/null
+++ b/flang/test/Preprocessing/exponent-bug.F90
@@ -0,0 +1,5 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: 3.14159e00
+#define e00 e666
+print *, 3.14159e00
+end
|
eugeneepshteyn
approved these changes
Apr 17, 2025
See new test. I inadvertently broke this behavior with a recent fix for another problem, because the effects of the overloaded TokenSequence::Put() member function on token merging were confusing. Rename and document the various overloads.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
See new test. I inadvertently broke this behavior with a recent fix for another problem, because the effects of the overloaded TokenSequence::Put() member function on token merging were confusing. Rename and document the various overloads.