Skip to content

Commit

Permalink
Extract codegen template to an embedded file.
Browse files Browse the repository at this point in the history
  • Loading branch information
@alecthomas
alecthomas committed Sep 27, 2022
1 parent a314b80 commit 0d264e9
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 151 deletions.
125 changes: 125 additions & 0 deletions cmd/participle/files/codegen.go.tmpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// Code generated by Participle. DO NOT EDIT.
package {{.Package}}

import (
"io"
"strings"
"unicode/utf8"
"regexp/syntax"

"github.com/alecthomas/participle/v2"
"github.com/alecthomas/participle/v2/lexer"
)

var _ syntax.Op

var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{}

type lexer{{.Name}}DefinitionImpl struct {}

func (lexer{{.Name}}DefinitionImpl) Symbols() map[string]lexer.TokenType {
return map[string]lexer.TokenType{
{{- range $sym, $rn := .Def.Symbols}}
"{{$sym}}": {{$rn}},
{{- end}}
}
}

func (lexer{{.Name}}DefinitionImpl) LexString(filename string, s string) (lexer.Lexer, error) {
return &lexer{{.Name}}Impl{
s: s,
pos: lexer.Position{
Filename: filename,
Line: 1,
Column: 1,
},
states: []lexer{{.Name}}State{ {name: "Root"} },
}, nil
}

func (d lexer{{.Name}}DefinitionImpl) LexBytes(filename string, b []byte) (lexer.Lexer, error) {
return d.LexString(filename, string(b))
}

func (d lexer{{.Name}}DefinitionImpl) Lex(filename string, r io.Reader) (lexer.Lexer, error) {
s := &strings.Builder{}
_, err := io.Copy(s, r)
if err != nil {
return nil, err
}
return d.LexString(filename, s.String())
}

type lexer{{.Name}}State struct {
name string
groups []string
}

type lexer{{.Name}}Impl struct {
s string
p int
pos lexer.Position
states []lexer{{.Name}}State
}

func (l *lexer{{.Name}}Impl) Next() (lexer.Token, error) {
if l.p == len(l.s) {
return lexer.EOFToken(l.pos), nil
}
var (
state = l.states[len(l.states)-1]
groups []int
sym lexer.TokenType
)
switch state.name {
{{- range $state := .Def.Rules|OrderRules}}
case "{{$state.Name}}":
{{- range $i, $rule := $state.Rules}}
{{- if $i}} else {{end -}}
{{- if .Pattern -}}
if match := match{{$.Name}}{{.Name}}(l.s, l.p); match[1] != 0 {
sym = {{index $.Def.Symbols .Name}}
groups = match[:]
{{- else if .|IsReturn -}}
if true {
{{- end}}
{{- if .|IsPush}}
l.states = append(l.states, lexer{{$.Name}}State{name: "{{.|IsPush}}"{{if HaveBackrefs $.Def $state.Name}}, groups: l.sgroups(groups){{end}}})
{{- else if (or (.|IsPop) (.|IsReturn))}}
l.states = l.states[:len(l.states)-1]
{{- if .|IsReturn}}
return l.Next()
{{- end}}
{{- else if not .Action}}
{{- else}}
Unsupported action {{.Action}}
{{- end}}
}
{{- end}}
{{- end}}
}
if groups == nil {
sample := []rune(l.s[l.p:])
if len(sample) > 16 {
sample = append(sample[:16], []rune("...")...)
}
return lexer.Token{}, participle.Errorf(l.pos, "invalid input text %q", sample)
}
pos := l.pos
span := l.s[groups[0]:groups[1]]
l.p = groups[1]
l.pos.Advance(span)
return lexer.Token{
Type: sym,
Value: span,
Pos: pos,
}, nil
}

func (l *lexer{{.Name}}Impl) sgroups(match []int) []string {
sgroups := make([]string, len(match)/2)
for i := 0; i < len(match)-1; i += 2 {
sgroups[i/2] = l.s[l.p+match[i]:l.p+match[i+1]]
}
return sgroups
}
179 changes: 28 additions & 151 deletions cmd/participle/gen_lexer_cmd.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
_ "embed" // For go:embed.
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -58,159 +59,35 @@ func (c *genLexerCmd) Run() error {
return nil
}

var codegenBackrefRe = regexp.MustCompile(`(\\+)(\d)`)

var codegenTemplate *template.Template = template.Must(template.New("lexgen").Funcs(template.FuncMap{
"IsPush": func(r lexer.Rule) string {
if p, ok := r.Action.(lexer.ActionPush); ok {
return p.State
}
return ""
},
"IsPop": func(r lexer.Rule) bool {
_, ok := r.Action.(lexer.ActionPop)
return ok
},
"IsReturn": func(r lexer.Rule) bool {
return r == lexer.ReturnRule
},
"OrderRules": orderRules,
"HaveBackrefs": func(def *lexer.StatefulDefinition, state string) bool {
for _, rule := range def.Rules()[state] {
if codegenBackrefRe.MatchString(rule.Pattern) {
return true
var (
//go:embed files/codegen.go.tmpl
codegenTemplateSource string
codegenBackrefRe = regexp.MustCompile(`(\\+)(\d)`)
codegenTemplate *template.Template = template.Must(template.New("lexgen").Funcs(template.FuncMap{
"IsPush": func(r lexer.Rule) string {
if p, ok := r.Action.(lexer.ActionPush); ok {
return p.State
}
}
return false
},
}).Parse(`
// Code generated by Participle. DO NOT EDIT.
package {{.Package}}
import (
"io"
"strings"
"unicode/utf8"
"regexp/syntax"
"github.com/alecthomas/participle/v2"
"github.com/alecthomas/participle/v2/lexer"
)
var _ syntax.Op
var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{}
type lexer{{.Name}}DefinitionImpl struct {}
func (lexer{{.Name}}DefinitionImpl) Symbols() map[string]lexer.TokenType {
return map[string]lexer.TokenType{
{{- range $sym, $rn := .Def.Symbols}}
"{{$sym}}": {{$rn}},
{{- end}}
}
}
func (lexer{{.Name}}DefinitionImpl) LexString(filename string, s string) (lexer.Lexer, error) {
return &lexer{{.Name}}Impl{
s: s,
pos: lexer.Position{
Filename: filename,
Line: 1,
Column: 1,
return ""
},
states: []lexer{{.Name}}State{ {name: "Root"} },
}, nil
}
func (d lexer{{.Name}}DefinitionImpl) LexBytes(filename string, b []byte) (lexer.Lexer, error) {
return d.LexString(filename, string(b))
}
func (d lexer{{.Name}}DefinitionImpl) Lex(filename string, r io.Reader) (lexer.Lexer, error) {
s := &strings.Builder{}
_, err := io.Copy(s, r)
if err != nil {
return nil, err
}
return d.LexString(filename, s.String())
}
type lexer{{.Name}}State struct {
name string
groups []string
}
type lexer{{.Name}}Impl struct {
s string
p int
pos lexer.Position
states []lexer{{.Name}}State
}
func (l *lexer{{.Name}}Impl) Next() (lexer.Token, error) {
if l.p == len(l.s) {
return lexer.EOFToken(l.pos), nil
}
var (
state = l.states[len(l.states)-1]
groups []int
sym lexer.TokenType
)
switch state.name {
{{- range $state := .Def.Rules|OrderRules}}
case "{{$state.Name}}":
{{- range $i, $rule := $state.Rules}}
{{- if $i}} else {{end -}}
{{- if .Pattern -}}
if match := match{{$.Name}}{{.Name}}(l.s, l.p); match[1] != 0 {
sym = {{index $.Def.Symbols .Name}}
groups = match[:]
{{- else if .|IsReturn -}}
if true {
{{- end}}
{{- if .|IsPush}}
l.states = append(l.states, lexer{{$.Name}}State{name: "{{.|IsPush}}"{{if HaveBackrefs $.Def $state.Name}}, groups: l.sgroups(groups){{end}}})
{{- else if (or (.|IsPop) (.|IsReturn))}}
l.states = l.states[:len(l.states)-1]
{{- if .|IsReturn}}
return l.Next()
{{- end}}
{{- else if not .Action}}
{{- else}}
Unsupported action {{.Action}}
{{- end}}
}
{{- end}}
{{- end}}
}
if groups == nil {
sample := []rune(l.s[l.p:])
if len(sample) > 16 {
sample = append(sample[:16], []rune("...")...)
}
return lexer.Token{}, participle.Errorf(l.pos, "invalid input text %q", sample)
}
pos := l.pos
span := l.s[groups[0]:groups[1]]
l.p = groups[1]
l.pos.Advance(span)
return lexer.Token{
Type: sym,
Value: span,
Pos: pos,
}, nil
}
func (l *lexer{{.Name}}Impl) sgroups(match []int) []string {
sgroups := make([]string, len(match)/2)
for i := 0; i < len(match)-1; i += 2 {
sgroups[i/2] = l.s[l.p+match[i]:l.p+match[i+1]]
}
return sgroups
}
`))
"IsPop": func(r lexer.Rule) bool {
_, ok := r.Action.(lexer.ActionPop)
return ok
},
"IsReturn": func(r lexer.Rule) bool {
return r == lexer.ReturnRule
},
"OrderRules": orderRules,
"HaveBackrefs": func(def *lexer.StatefulDefinition, state string) bool {
for _, rule := range def.Rules()[state] {
if codegenBackrefRe.MatchString(rule.Pattern) {
return true
}
}
return false
},
}).Parse(codegenTemplateSource))
)

func generateLexer(w io.Writer, pkg string, def *lexer.StatefulDefinition, name string) error {
type ctx struct {
Expand Down

0 comments on commit 0d264e9

Please sign in to comment.