Skip to content

[WIP] [Parser] Decouple the parser from AST creation (part 3.1) #26738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[WIP] [Parser] Decouple the parser from AST creation (part 3.1) #26738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions include/swift/Parse/ASTGen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,27 +23,36 @@
namespace swift {
/// Generates AST nodes from Syntax nodes.
class ASTGen {
// todo[gsoc]: use only members of Parser that are needed for AST generation
Parser &P;

ASTContext &Context;

/// Type cache to prevent multiple transformations of the same syntax node.
llvm::DenseMap<syntax::SyntaxNodeId, TypeRepr *> TypeCache;

PersistentParserState **ParserState;

// FIXME: remove when Syntax can represent all types and ASTGen can handle them
/// Types that cannot be represented by Syntax or generated by ASTGen.
llvm::DenseMap<SourceLoc, TypeRepr *> Types;

/// Expressions that cannot be created by ASTGen yet.
llvm::DenseMap<const void *, Expr *> Exprs;
public:
ASTGen(ASTContext &Context, PersistentParserState **ParserState)
: Context(Context), ParserState(ParserState) {}
ASTGen(Parser& P, ASTContext &Context) : P(P), Context(Context) {}

SourceLoc generate(syntax::TokenSyntax Tok, SourceLoc &Loc);

Expr *generate(syntax::ExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::IntegerLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::FloatLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::NilLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::BooleanLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::IsExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::AsExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::ArrowExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::ArrayExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::DictionaryExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::SuperRefExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundFileExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundLineExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundColumnExprSyntax &Expr, SourceLoc &Loc);
Expand Down Expand Up @@ -83,6 +92,15 @@ class ASTGen {
syntax::TokenSyntax RParen, SourceLoc &Loc,
bool IsFunction = false);

void addArrayElement(syntax::ArrayElementSyntax Element, SourceLoc &Loc,
llvm::SmallVectorImpl<Expr *> &ElementExprs,
llvm::SmallVectorImpl<SourceLoc> &CommaLocs);

void addDictionaryElement(syntax::DictionaryElementSyntax Element,
SourceLoc &Loc,
llvm::SmallVectorImpl<Expr *> &ElementExprs,
llvm::SmallVectorImpl<SourceLoc> &CommaLocs);

void gatherTypeIdentifierComponents(
syntax::TypeSyntax Component, SourceLoc &Loc,
llvm::SmallVectorImpl<ComponentIdentTypeRepr *> &Components);
Expand All @@ -105,17 +123,19 @@ class ASTGen {
static MagicIdentifierLiteralExpr::Kind getMagicIdentifierLiteralKind(tok Kind);

ValueDecl *lookupInScope(DeclName Name);
VarDecl *getImplicitSelfDeclForSuperContext(SourceLoc Loc);

TypeRepr *cacheType(syntax::TypeSyntax Type, TypeRepr *TypeAST);

TypeRepr *lookupType(syntax::TypeSyntax Type);

public:
TypeRepr *addType(TypeRepr *Type, const SourceLoc &Loc);

bool hasType(const SourceLoc &Loc) const;

TypeRepr *getType(const SourceLoc &Loc) const;

Expr *addExpr(Expr* E, const SourceLoc &Loc);
bool hasExpr(const SourceLoc &Loc) const;
Expr *getExpr(const SourceLoc &Loc) const;
};
} // namespace swift

Expand Down
28 changes: 21 additions & 7 deletions include/swift/Parse/Parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,6 +624,12 @@ class Parser {
return consumeTokenSyntax();
}

llvm::Optional<ParsedTokenSyntax> consumeTokenSyntaxIfAny(tok K1, tok K2) {
if (Tok.is(K1) || Tok.is(K2))
return consumeTokenSyntax();
return llvm::None;
}

/// If the current token is the specified kind, consume it and
/// return true. Otherwise, return false without consuming it.
bool consumeIf(tok K) {
Expand Down Expand Up @@ -1398,16 +1404,22 @@ class Parser {

//===--------------------------------------------------------------------===//
// Expression Parsing

using ExprResult = ParsedSyntaxResult<ParsedExprSyntax>;

ParserResult<Expr> parseExpr(Diag<> ID) {
return parseExprImpl(ID, /*isExprBasic=*/false);
}
ParserResult<Expr> parseExprBasic(Diag<> ID) {
return parseExprImpl(ID, /*isExprBasic=*/true);
}
ParserResult<Expr> parseExprImpl(Diag<> ID, bool isExprBasic);
ParserResult<Expr> parseExprIs();
ParserResult<Expr> parseExprAs();
ParserResult<Expr> parseExprArrow();
ExprResult parseExprIs();
ExprResult parseExprAs();
ExprResult parseExprArrow();
ParserResult<Expr> parseExprIsAST();
ParserResult<Expr> parseExprAsAST();
ParserResult<Expr> parseExprArrowAST();
ParserResult<Expr> parseExprSequence(Diag<> ID,
bool isExprBasic,
bool isForConditionalDirective = false);
Expand All @@ -1423,7 +1435,8 @@ class Parser {
ParserResult<Expr> parseExprKeyPathObjC();
ParserResult<Expr> parseExprKeyPath();
ParserResult<Expr> parseExprSelector();
ParserResult<Expr> parseExprSuper();
ParsedResult<ParsedExprSyntax> parseExprSuper();
ParserResult<Expr> parseExprSuperAST();
ParserResult<Expr> parseExprStringLiteral();

// todo [gsoc]: create new result type for ParsedSyntax
Expand Down Expand Up @@ -1538,15 +1551,16 @@ class Parser {
bool isExprBasic);
ParserResult<Expr> parseExprCallSuffix(ParserResult<Expr> fn,
bool isExprBasic);
ParserResult<Expr> parseExprCollection();
ParserResult<Expr> parseExprCollectionElement(Optional<bool> &isDictionary);
ParsedResult<ParsedExprSyntax> parseExprCollection();
ParsedResult<> parseExprCollectionElement(Optional<bool> &IsDict, bool &SeenComma);
ParserResult<Expr> parseExprCollectionAST();
ParserResult<Expr> parseExprPoundUnknown(SourceLoc LSquareLoc);
ParserResult<Expr>
parseExprPoundCodeCompletion(Optional<StmtKind> ParentKind);

UnresolvedDeclRefExpr *parseExprOperator();

void validateCollectionElement(ParserResult<Expr> element);
void validateCollectionElement(ParsedArrayElementSyntax Element, SourceLoc Loc);

//===--------------------------------------------------------------------===//
// Statement Parsing
Expand Down
80 changes: 79 additions & 1 deletion include/swift/Parse/SyntaxParserResult.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@
#ifndef SWIFT_PARSE_SYNTAXPARSERRESULT_H
#define SWIFT_PARSE_SYNTAXPARSERRESULT_H

#include "llvm/ADT/Optional.h"
#include "swift/Parse/ParserResult.h"
#include "swift/Parse/ParsedSyntaxRecorder.h"
#include "llvm/ADT/Optional.h"

namespace swift {

Expand All @@ -24,6 +25,83 @@ enum class ResultDataKind : uint8_t {
CodeCompletion,
};

template <typename NodeType = ParsedSyntax>
class ParsedResult {
private:
SmallVector<ParsedSyntax, 8> Nodes;
ParserStatus Status;

template <typename O>
friend class ParsedResult;

public:
ParsedResult() {
Status.setIsParseError();
}

bool isSuccess() const { return Status.isSuccess(); }
bool isError() const { return Status.isError(); }
bool hasCodeCompletion() const { return Status.hasCodeCompletion(); }
ParserStatus getStatus() const { return Status; }
SmallVectorImpl<ParsedSyntax>& getStorage() { return Nodes; }

template <typename T = NodeType>
T get(std::size_t i = 0) const {
return Nodes[i].template castTo<T>();
}

template <typename T = NodeType>
llvm::Optional<T> maybeGet(std::size_t i = 0) const {
if (Nodes.size() <= i)
return None;
return Nodes[i].template getAs<T>();
}

ArrayRef<ParsedSyntax> getNodes() const {
return Nodes;
}

template <typename T>
void add(T Node) {
Nodes.push_back(Node);
}

template <typename T>
void add(Optional<T> Node) {
if (Node)
Nodes.push_back(*Node);
}

template <typename O>
ParsedResult<NodeType> &append(ParsedResult<O> &Other) {
Nodes.insert(Nodes.end(), Other.Nodes.begin(), Other.Nodes.end());
return *this;
}

ParsedResult<NodeType> &with(NodeType Node, ParserStatus NewStatus = ParserStatus()) {
Nodes = {Node};
Status = NewStatus;
return *this;
}

ParsedResult<NodeType> &withUnknownExpr(SyntaxParsingContext *Ctx) {
Nodes = {ParsedSyntaxRecorder::makeUnknownExpr(Nodes, *Ctx)};
Status.setIsParseError();
return *this;
}

ParsedResult<NodeType> &withUnknownExpr(SyntaxParsingContext *Ctx, ParserStatus NewStatus) {
Nodes = {ParsedSyntaxRecorder::makeUnknownExpr(Nodes, *Ctx)};
Status = NewStatus;
return *this;
}

ParsedResult<NodeType> &as(ParserStatus NewStatus) {
Status = NewStatus;
return *this;
}
};

template <typename ParsedSyntaxNode> class ParsedSyntaxResult {
public:
template <typename OtherParsedSyntaxNode>
Expand Down
Loading