From e44e4b930be3151f2298f80e0bb12ae71fdeef2d Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Tue, 28 Feb 2023 17:05:47 +1100 Subject: [PATCH] Upgrade go+golangci-lint and fix lint rules --- .golangci.yml | 3 +- _examples/thrift/lexer_gen.go | 469 ++++++++++-------- bin/{.go-1.19.2.pkg => .go-1.20.1.pkg} | 0 ...t-1.46.2.pkg => .golangci-lint-1.51.2.pkg} | 0 bin/go | 2 +- bin/gofmt | 2 +- bin/golangci-lint | 2 +- doc.go | 92 ++-- ebnf/ebnf.go | 12 +- lexer/api.go | 2 +- lexer/stateful_test.go | 24 + lookahead_test.go | 20 +- options.go | 4 +- 13 files changed, 367 insertions(+), 265 deletions(-) rename bin/{.go-1.19.2.pkg => .go-1.20.1.pkg} (100%) rename bin/{.golangci-lint-1.46.2.pkg => .golangci-lint-1.51.2.pkg} (100%) diff --git a/.golangci.yml b/.golangci.yml index 5229d875..54161bc6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,6 +51,7 @@ linters: - nilnil - maintidx - unused # Does not work with type parameters + - dupword linters-settings: govet: @@ -75,7 +76,7 @@ issues: # Very commonly not checked. - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked' - 'exported method `(.*\.MarshalJSON|.*\.UnmarshalJSON|.*\.EntityURN|.*\.GoString|.*\.Pos)` should have comment or be unexported' - - 'composite literal uses unkeyed fields' + - 'uses unkeyed fields' - 'declaration of "err" shadows declaration' - 'bad syntax for struct tag key' - 'bad syntax for struct tag pair' diff --git a/_examples/thrift/lexer_gen.go b/_examples/thrift/lexer_gen.go index 18bcc616..f733d27f 100644 --- a/_examples/thrift/lexer_gen.go +++ b/_examples/thrift/lexer_gen.go @@ -1,12 +1,11 @@ - // Code generated by Participle. DO NOT EDIT. package main import ( "io" + "regexp/syntax" "strings" "unicode/utf8" - "regexp/syntax" "github.com/alecthomas/participle/v2" "github.com/alecthomas/participle/v2/lexer" @@ -16,17 +15,17 @@ var _ syntax.Op var Lexer lexer.Definition = definitionImpl{} -type definitionImpl struct {} +type definitionImpl struct{} func (definitionImpl) Symbols() map[string]lexer.TokenType { return map[string]lexer.TokenType{ - "Comment": -7, - "EOF": -1, - "Ident": -3, - "Number": -2, - "Punct": -6, - "String": -4, - "Whitespace": -5, + "Comment": -7, + "EOF": -1, + "Ident": -3, + "Number": -2, + "Punct": -6, + "String": -4, + "Whitespace": -5, } } @@ -38,7 +37,7 @@ func (definitionImpl) LexString(filename string, s string) (lexer.Lexer, error) Line: 1, Column: 1, }, - states: []lexerState{lexerState{name: "Root"}}, + states: []lexerState{{name: "Root"}}, }, nil } @@ -56,15 +55,15 @@ func (d definitionImpl) Lex(filename string, r io.Reader) (lexer.Lexer, error) { } type lexerState struct { - name string - groups []string + name string + groups []string } type lexerImpl struct { - s string - p int - pos lexer.Position - states []lexerState + s string + p int + pos lexer.Position + states []lexerState } func (l *lexerImpl) Next() (lexer.Token, error) { @@ -72,12 +71,13 @@ func (l *lexerImpl) Next() (lexer.Token, error) { return lexer.EOFToken(l.pos), nil } var ( - state = l.states[len(l.states)-1] + state = l.states[len(l.states)-1] groups []int - sym lexer.TokenType + sym lexer.TokenType ) switch state.name { - case "Root":if match := matchNumber(l.s, l.p); match[1] != 0 { + case "Root": + if match := matchNumber(l.s, l.p); match[1] != 0 { sym = -2 groups = match[:] } else if match := matchIdent(l.s, l.p); match[1] != 0 { @@ -118,207 +118,282 @@ func (l *lexerImpl) Next() (lexer.Token, error) { func (l *lexerImpl) sgroups(match []int) []string { sgroups := make([]string, len(match)/2) for i := 0; i < len(match)-1; i += 2 { - sgroups[i/2] = l.s[l.p+match[i]:l.p+match[i+1]] + sgroups[i/2] = l.s[l.p+match[i] : l.p+match[i+1]] } return sgroups } // [0-9]+ func matchNumber(s string, p int) (groups [2]int) { -// [0-9] (CharClass) -l0 := func(s string, p int) int { -if len(s) <= p { return -1 } -rn := s[p] -switch { -case rn >= '0' && rn <= '9': return p+1 -} -return -1 -} -// [0-9]+ (Plus) -l1 := func(s string, p int) int { -if p = l0(s, p); p == -1 { return -1 } -for len(s) > p { -if np := l0(s, p); np == -1 { return p } else { p = np } -} -return p -} -np := l1(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // [0-9] (CharClass) + l0 := func(s string, p int) int { + if len(s) <= p { + return -1 + } + rn := s[p] + switch { + case rn >= '0' && rn <= '9': + return p + 1 + } + return -1 + } + // [0-9]+ (Plus) + l1 := func(s string, p int) int { + if p = l0(s, p); p == -1 { + return -1 + } + for len(s) > p { + if np := l0(s, p); np == -1 { + return p + } else { + p = np + } + } + return p + } + np := l1(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } // [0-9A-Z_a-z]+ func matchIdent(s string, p int) (groups [2]int) { -// [0-9A-Z_a-z] (CharClass) -l0 := func(s string, p int) int { -if len(s) <= p { return -1 } -rn := s[p] -switch { -case rn >= '0' && rn <= '9': return p+1 -case rn >= 'A' && rn <= 'Z': return p+1 -case rn == '_': return p+1 -case rn >= 'a' && rn <= 'z': return p+1 -} -return -1 -} -// [0-9A-Z_a-z]+ (Plus) -l1 := func(s string, p int) int { -if p = l0(s, p); p == -1 { return -1 } -for len(s) > p { -if np := l0(s, p); np == -1 { return p } else { p = np } -} -return p -} -np := l1(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // [0-9A-Z_a-z] (CharClass) + l0 := func(s string, p int) int { + if len(s) <= p { + return -1 + } + rn := s[p] + switch { + case rn >= '0' && rn <= '9': + return p + 1 + case rn >= 'A' && rn <= 'Z': + return p + 1 + case rn == '_': + return p + 1 + case rn >= 'a' && rn <= 'z': + return p + 1 + } + return -1 + } + // [0-9A-Z_a-z]+ (Plus) + l1 := func(s string, p int) int { + if p = l0(s, p); p == -1 { + return -1 + } + for len(s) > p { + if np := l0(s, p); np == -1 { + return p + } else { + p = np + } + } + return p + } + np := l1(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } // "[^"]*" func matchString(s string, p int) (groups [2]int) { -// " (Literal) -l0 := func(s string, p int) int { -if p < len(s) && s[p] == '"' { return p+1 } -return -1 -} -// [^"] (CharClass) -l1 := func(s string, p int) int { -if len(s) <= p { return -1 } -var (rn rune; n int) -if s[p] < utf8.RuneSelf { - rn, n = rune(s[p]), 1 -} else { - rn, n = utf8.DecodeRuneInString(s[p:]) -} -switch { -case rn >= '\x00' && rn <= '!': return p+1 -case rn >= '#' && rn <= '\U0010ffff': return p+n -} -return -1 -} -// [^"]* (Star) -l2 := func(s string, p int) int { -for len(s) > p { -if np := l1(s, p); np == -1 { return p } else { p = np } -} -return p -} -// "[^"]*" (Concat) -l3 := func(s string, p int) int { -if p = l0(s, p); p == -1 { return -1 } -if p = l2(s, p); p == -1 { return -1 } -if p = l0(s, p); p == -1 { return -1 } -return p -} -np := l3(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // " (Literal) + l0 := func(s string, p int) int { + if p < len(s) && s[p] == '"' { + return p + 1 + } + return -1 + } + // [^"] (CharClass) + l1 := func(s string, p int) int { + if len(s) <= p { + return -1 + } + var ( + rn rune + n int + ) + if s[p] < utf8.RuneSelf { + rn, n = rune(s[p]), 1 + } else { + rn, n = utf8.DecodeRuneInString(s[p:]) + } + switch { + case rn >= '\x00' && rn <= '!': + return p + 1 + case rn >= '#' && rn <= '\U0010ffff': + return p + n + } + return -1 + } + // [^"]* (Star) + l2 := func(s string, p int) int { + for len(s) > p { + if np := l1(s, p); np == -1 { + return p + } else { + p = np + } + } + return p + } + // "[^"]*" (Concat) + l3 := func(s string, p int) int { + if p = l0(s, p); p == -1 { + return -1 + } + if p = l2(s, p); p == -1 { + return -1 + } + if p = l0(s, p); p == -1 { + return -1 + } + return p + } + np := l3(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } // [\t-\n\f-\r ]+ func matchWhitespace(s string, p int) (groups [2]int) { -// [\t-\n\f-\r ] (CharClass) -l0 := func(s string, p int) int { -if len(s) <= p { return -1 } -rn := s[p] -switch { -case rn >= '\t' && rn <= '\n': return p+1 -case rn >= '\f' && rn <= '\r': return p+1 -case rn == ' ': return p+1 -} -return -1 -} -// [\t-\n\f-\r ]+ (Plus) -l1 := func(s string, p int) int { -if p = l0(s, p); p == -1 { return -1 } -for len(s) > p { -if np := l0(s, p); np == -1 { return p } else { p = np } -} -return p -} -np := l1(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // [\t-\n\f-\r ] (CharClass) + l0 := func(s string, p int) int { + if len(s) <= p { + return -1 + } + rn := s[p] + switch { + case rn >= '\t' && rn <= '\n': + return p + 1 + case rn >= '\f' && rn <= '\r': + return p + 1 + case rn == ' ': + return p + 1 + } + return -1 + } + // [\t-\n\f-\r ]+ (Plus) + l1 := func(s string, p int) int { + if p = l0(s, p); p == -1 { + return -1 + } + for len(s) > p { + if np := l0(s, p); np == -1 { + return p + } else { + p = np + } + } + return p + } + np := l1(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } // [\(-\),\.:<->\{\}] func matchPunct(s string, p int) (groups [2]int) { -// [\(-\),\.:<->\{\}] (CharClass) -l0 := func(s string, p int) int { -if len(s) <= p { return -1 } -rn := s[p] -switch { -case rn >= '(' && rn <= ')': return p+1 -case rn == ',': return p+1 -case rn == '.': return p+1 -case rn == ':': return p+1 -case rn >= '<' && rn <= '>': return p+1 -case rn == '{': return p+1 -case rn == '}': return p+1 -} -return -1 -} -np := l0(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // [\(-\),\.:<->\{\}] (CharClass) + l0 := func(s string, p int) int { + if len(s) <= p { + return -1 + } + rn := s[p] + switch { + case rn >= '(' && rn <= ')': + return p + 1 + case rn == ',': + return p + 1 + case rn == '.': + return p + 1 + case rn == ':': + return p + 1 + case rn >= '<' && rn <= '>': + return p + 1 + case rn == '{': + return p + 1 + case rn == '}': + return p + 1 + } + return -1 + } + np := l0(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } // //(?-s:.)* func matchComment(s string, p int) (groups [2]int) { -// // (Literal) -l0 := func(s string, p int) int { -if p+2 < len(s) && s[p:p+2] == "//" { return p+2 } -return -1 -} -// (?-s:.) (AnyCharNotNL) -l1 := func(s string, p int) int { -var (rn rune; n int) -if s[p] < utf8.RuneSelf { - rn, n = rune(s[p]), 1 -} else { - rn, n = utf8.DecodeRuneInString(s[p:]) -} -if len(s) <= p+n || rn == '\n' { return -1 } -return p+n -} -// (?-s:.)* (Star) -l2 := func(s string, p int) int { -for len(s) > p { -if np := l1(s, p); np == -1 { return p } else { p = np } -} -return p -} -// //(?-s:.)* (Concat) -l3 := func(s string, p int) int { -if p = l0(s, p); p == -1 { return -1 } -if p = l2(s, p); p == -1 { return -1 } -return p -} -np := l3(s, p) -if np == -1 { - return -} -groups[0] = p -groups[1] = np -return + // // (Literal) + l0 := func(s string, p int) int { + if p+2 < len(s) && s[p:p+2] == "//" { + return p + 2 + } + return -1 + } + // (?-s:.) (AnyCharNotNL) + l1 := func(s string, p int) int { + var ( + rn rune + n int + ) + if s[p] < utf8.RuneSelf { + rn, n = rune(s[p]), 1 + } else { + rn, n = utf8.DecodeRuneInString(s[p:]) + } + if len(s) <= p+n || rn == '\n' { + return -1 + } + return p + n + } + // (?-s:.)* (Star) + l2 := func(s string, p int) int { + for len(s) > p { + if np := l1(s, p); np == -1 { + return p + } else { + p = np + } + } + return p + } + // //(?-s:.)* (Concat) + l3 := func(s string, p int) int { + if p = l0(s, p); p == -1 { + return -1 + } + if p = l2(s, p); p == -1 { + return -1 + } + return p + } + np := l3(s, p) + if np == -1 { + return + } + groups[0] = p + groups[1] = np + return } diff --git a/bin/.go-1.19.2.pkg b/bin/.go-1.20.1.pkg similarity index 100% rename from bin/.go-1.19.2.pkg rename to bin/.go-1.20.1.pkg diff --git a/bin/.golangci-lint-1.46.2.pkg b/bin/.golangci-lint-1.51.2.pkg similarity index 100% rename from bin/.golangci-lint-1.46.2.pkg rename to bin/.golangci-lint-1.51.2.pkg diff --git a/bin/go b/bin/go index 50baf175..f87f6f79 120000 --- a/bin/go +++ b/bin/go @@ -1 +1 @@ -.go-1.19.2.pkg \ No newline at end of file +.go-1.20.1.pkg \ No newline at end of file diff --git a/bin/gofmt b/bin/gofmt index 50baf175..f87f6f79 120000 --- a/bin/gofmt +++ b/bin/gofmt @@ -1 +1 @@ -.go-1.19.2.pkg \ No newline at end of file +.go-1.20.1.pkg \ No newline at end of file diff --git a/bin/golangci-lint b/bin/golangci-lint index de42e349..7d8886c7 120000 --- a/bin/golangci-lint +++ b/bin/golangci-lint @@ -1 +1 @@ -.golangci-lint-1.46.2.pkg \ No newline at end of file +.golangci-lint-1.51.2.pkg \ No newline at end of file diff --git a/doc.go b/doc.go index 48d003dc..da128839 100644 --- a/doc.go +++ b/doc.go @@ -4,65 +4,65 @@ // // The supported annotation syntax is: // -// - `@` Capture expression into the field. -// - `@@` Recursively capture using the fields own type. -// - `` Match named lexer token. -// - `( ... )` Group. -// - `"..."` Match the literal (note that the lexer must emit tokens matching this literal exactly). -// - `"...":` Match the literal, specifying the exact lexer token type to match. -// - ` ...` Match expressions. -// - ` | ` Match one of the alternatives. +// - `@` Capture expression into the field. +// - `@@` Recursively capture using the fields own type. +// - `` Match named lexer token. +// - `( ... )` Group. +// - `"..."` Match the literal (note that the lexer must emit tokens matching this literal exactly). +// - `"...":` Match the literal, specifying the exact lexer token type to match. +// - ` ...` Match expressions. +// - ` | ` Match one of the alternatives. // // The following modifiers can be used after any expression: // -// - `*` Expression can match zero or more times. -// - `+` Expression must match one or more times. -// - `?` Expression can match zero or once. -// - `!` Require a non-empty match (this is useful with a sequence of optional matches eg. `("a"? "b"? "c"?)!`). +// - `*` Expression can match zero or more times. +// - `+` Expression must match one or more times. +// - `?` Expression can match zero or once. +// - `!` Require a non-empty match (this is useful with a sequence of optional matches eg. `("a"? "b"? "c"?)!`). // // Here's an example of an EBNF grammar. // -// type Group struct { -// Expression *Expression `"(" @@ ")"` -// } +// type Group struct { +// Expression *Expression `"(" @@ ")"` +// } // -// type Option struct { -// Expression *Expression `"[" @@ "]"` -// } +// type Option struct { +// Expression *Expression `"[" @@ "]"` +// } // -// type Repetition struct { -// Expression *Expression `"{" @@ "}"` -// } +// type Repetition struct { +// Expression *Expression `"{" @@ "}"` +// } // -// type Literal struct { -// Start string `@String` // lexer.Lexer token "String" -// End string `("…" @String)?` -// } +// type Literal struct { +// Start string `@String` // lexer.Lexer token "String" +// End string `("…" @String)?` +// } // -// type Term struct { -// Name string ` @Ident` -// Literal *Literal `| @@` -// Group *Group `| @@` -// Option *Option `| @@` -// Repetition *Expression `| "(" @@ ")"` -// } +// type Term struct { +// Name string ` @Ident` +// Literal *Literal `| @@` +// Group *Group `| @@` +// Option *Option `| @@` +// Repetition *Expression `| "(" @@ ")"` +// } // -// type Sequence struct { -// Terms []*Term `@@+` -// } +// type Sequence struct { +// Terms []*Term `@@+` +// } // -// type Expression struct { -// Alternatives []*Sequence `@@ ("|" @@)*` -// } +// type Expression struct { +// Alternatives []*Sequence `@@ ("|" @@)*` +// } // -// type Expressions []*Expression +// type Expressions []*Expression // -// type Production struct { -// Name string `@Ident "="` -// Expressions Expressions `@@+ "."` -// } +// type Production struct { +// Name string `@Ident "="` +// Expressions Expressions `@@+ "."` +// } // -// type EBNF struct { -// Productions []*Production `@@*` -// } +// type EBNF struct { +// Productions []*Production `@@*` +// } package participle diff --git a/ebnf/ebnf.go b/ebnf/ebnf.go index 30088a44..4710418a 100644 --- a/ebnf/ebnf.go +++ b/ebnf/ebnf.go @@ -2,12 +2,12 @@ // // The self-referential EBNF is: // -// EBNF = Production* . -// Production = "=" Expression "." . -// Expression = Sequence ("|" Sequence)* . -// SubExpression = "(" ("?!" | "?=")? Expression ")" . -// Sequence = Term+ . -// Term = "~"? ( | | ("<" ">") | SubExpression) ("*" | "+" | "?" | "!")? . +// EBNF = Production* . +// Production = "=" Expression "." . +// Expression = Sequence ("|" Sequence)* . +// SubExpression = "(" ("?!" | "?=")? Expression ")" . +// Sequence = Term+ . +// Term = "~"? ( | | ("<" ">") | SubExpression) ("*" | "+" | "?" | "!")? . package ebnf import ( diff --git a/lexer/api.go b/lexer/api.go index 9f89d7aa..5e732f5c 100644 --- a/lexer/api.go +++ b/lexer/api.go @@ -70,7 +70,7 @@ func NameOfReader(r interface{}) string { // // eg. // -// lex = lexer.Must(lexer.Build(`Symbol = "symbol" .`)) +// lex = lexer.Must(lexer.Build(`Symbol = "symbol" .`)) func Must(def Definition, err error) Definition { if err != nil { panic(err) diff --git a/lexer/stateful_test.go b/lexer/stateful_test.go index 31ef6128..2c0d2a0d 100644 --- a/lexer/stateful_test.go +++ b/lexer/stateful_test.go @@ -243,6 +243,30 @@ func ExampleNew() { log.Fatal(err) } repr.Println(actual) + // Output: &lexer_test.String{ + // Fragments: []*lexer_test.Fragment{ + // { + // Text: "hello ", + // }, + // { + // Expr: &lexer_test.Expr{ + // Left: &lexer_test.Terminal{ + // Ident: "user", + // }, + // Op: "+", + // Right: &lexer_test.Terminal{ + // String: &lexer_test.String{ + // Fragments: []*lexer_test.Fragment{ + // { + // Text: "??", + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // } } type String struct { diff --git a/lookahead_test.go b/lookahead_test.go index 69110001..e90b6d36 100644 --- a/lookahead_test.go +++ b/lookahead_test.go @@ -255,16 +255,16 @@ func TestIssue28(t *testing.T) { // // eg. // -// 0. groups = [ -// {history: [">"] roots: [0, 1]}, -// {history: ["<"], roots: [2, 3]}, -// ] -// 1. groups = [ -// {history: [">", "="], roots: [0]}, -// {history: [">"], roots: [1]}, -// {history: ["<", "="], roots: [2]}, -// {history: ["<"], roots: [3]}, -// ] +// 0. groups = [ +// {history: [">"] roots: [0, 1]}, +// {history: ["<"], roots: [2, 3]}, +// ] +// 1. groups = [ +// {history: [">", "="], roots: [0]}, +// {history: [">"], roots: [1]}, +// {history: ["<", "="], roots: [2]}, +// {history: ["<"], roots: [3]}, +// ] func TestLookaheadWithConvergingTokens(t *testing.T) { type grammar struct { Left string `@Ident` diff --git a/options.go b/options.go index 4842ecba..396eeb85 100644 --- a/options.go +++ b/options.go @@ -84,7 +84,9 @@ func ParseTypeWith[T any](parseFn func(*lexer.PeekingLexer) (T, error)) Option { // Union associates several member productions with some interface type T. // Given members X, Y, Z, and W for a union type U, then the EBNF rule is: -// U = X | Y | Z | W . +// +// U = X | Y | Z | W . +// // When the parser encounters a field of type T, it will attempt to parse each member // in sequence and take the first match. Because of this, the order in which the // members are defined is important. You must be careful to order your members appropriately.