Merge pull request #13 from KeisukeYamashita/add-comment

Add comment field

README.md

@@ -73,6 +73,7 @@ There are struct tags you can use for you input.
 * `block`: Represents a unit of your block like `acl`, `sub`, etc...
 * `label`: The label of your block.
 * `flat`: Represents a expression field
+* `comment`: Get comments
 * `attr`: (Default) Attribute of your block
 
 ## Releases

internal/ast/ast.go

@@ -120,6 +120,16 @@ func (as *ReturnStatement) TokenLiteral() string {
 	return as.Token.Literal
 }
 
+type CommentStatement struct {
+	Token token.Token
+	Value string
+}
+
+func (as *CommentStatement) statementNode() {}
+func (as *CommentStatement) TokenLiteral() string {
+	return as.Token.Literal
+}
+
 // CallStatement holds the Name for the Identifier and its value
 type CallStatement struct {
 	Token     token.Token // token.ASSIGN

internal/decoder/decoder.go

@@ -159,6 +159,29 @@ func decodeProgramToStruct(program *ast.Program, val reflect.Value) []error {
 		}
 	}
 
+	for _, n := range tags.Comments {
+		comments := content.Comments
+		field := val.Type().Field(n.FieldIndex)
+		fieldTy := field.Type
+
+		var isSlice bool
+		if fieldTy.Kind() == reflect.Slice {
+			isSlice = true
+			fieldTy = fieldTy.Elem()
+		}
+
+		switch {
+		case isSlice:
+			sli := reflect.MakeSlice(reflect.SliceOf(fieldTy), len(comments), len(comments))
+
+			for i, comment := range comments {
+				sli.Index(i).Set(reflect.ValueOf(comment))
+			}
+
+			val.Field(n.FieldIndex).Set(sli)
+		}
+	}
+
 	return nil
 }
 
@@ -298,6 +321,29 @@ func decodeBlockToStruct(block *schema.Block, val reflect.Value) {
 		}
 	}
 
+	for _, n := range tags.Comments {
+		comments := content.Comments
+		field := val.Type().Field(n.FieldIndex)
+		fieldTy := field.Type
+
+		var isSlice bool
+		if fieldTy.Kind() == reflect.Slice {
+			isSlice = true
+			fieldTy = fieldTy.Elem()
+		}
+
+		switch {
+		case isSlice:
+			sli := reflect.MakeSlice(reflect.SliceOf(fieldTy), len(comments), len(comments))
+
+			for i, comment := range comments {
+				sli.Index(i).Set(reflect.ValueOf(comment))
+			}
+
+			val.Field(n.FieldIndex).Set(sli)
+		}
+	}
+
 	return
 }
 
@@ -389,6 +435,7 @@ type fieldTags struct {
 	Blocks     map[string]int
 	Labels     []labelField
 	Flats      []flatField
+	Comments   []commentField
 }
 
 // labelField is a struct that represents info about the struct tags of "vcl".
@@ -401,13 +448,19 @@ type flatField struct {
 	Name       string
 }
 
+type commentField struct {
+	FieldIndex int
+	Name       string
+}
+
 // getFieldTags retrieves the "vcl" tags of the given struct type.
 func getFieldTags(ty reflect.Type) *fieldTags {
 	ret := &fieldTags{
 		Attributes: map[string]int{},
 		Blocks:     map[string]int{},
 		Labels:     []labelField{},
 		Flats:      []flatField{},
+		Comments:   []commentField{},
 	}
 
 	ct := ty.NumField()
@@ -443,6 +496,11 @@ func getFieldTags(ty reflect.Type) *fieldTags {
 				FieldIndex: i,
 				Name:       name,
 			})
+		case "comment":
+			ret.Comments = append(ret.Comments, commentField{
+				FieldIndex: i,
+				Name:       name,
+			})
 		default:
 			panic(fmt.Sprintf("invalid vcl field tag kind %q on %s %q", kind, field.Type.String(), field.Name))
 		}

internal/decoder/decoder_test.go

@@ -330,6 +330,63 @@ func TestDecodeProgramToStruct_NestedBlock(t *testing.T) {
 	}
 }
 
+func TestDecodeProgramToStruct_Comments(t *testing.T) {
+	type ACL struct {
+		Type     string   `vcl:"type,label"`
+		Comments []string `vcl:",comment"`
+	}
+
+	type Root struct {
+		ACLs     []*ACL   `vcl:"acl,block"`
+		Comments []string `vcl:",comment"`
+	}
+
+	testCases := map[string]struct {
+		input    string
+		val      interface{}
+		expected interface{}
+	}{
+		"with root comment by hash": {
+			`# keke`, &Root{}, &Root{Comments: []string{"keke"}, ACLs: []*ACL{}},
+		},
+		"with root comment by double slash": {
+			`// keke`, &Root{}, &Root{Comments: []string{"keke"}, ACLs: []*ACL{}},
+		},
+		"with root by double slash with block": {
+			`// keke
+acl "tag" {}		
+`, &Root{}, &Root{Comments: []string{"keke"}, ACLs: []*ACL{&ACL{Type: "tag", Comments: []string{}}}},
+		},
+		"with nested block": {
+			`// keke
+acl "tag" {
+	// internal-keke
+	"localhost";
+}		
+`, &Root{}, &Root{Comments: []string{"keke"}, ACLs: []*ACL{&ACL{Type: "tag", Comments: []string{"internal-keke"}}}},
+		},
+	}
+
+	for n, tc := range testCases {
+		t.Run(n, func(t *testing.T) {
+			l := lexer.NewLexer(tc.input)
+			p := parser.NewParser(l)
+			program := p.ParseProgram()
+			root := tc.val
+			val := reflect.ValueOf(root).Elem()
+			errs := decodeProgramToStruct(program, val)
+
+			if len(errs) > 0 {
+				t.Fatalf("decodeProgramToStruct_Block has errorr, err:%v", errs)
+			}
+
+			if !reflect.DeepEqual(tc.val, tc.expected) {
+				t.Fatalf("decodeProgramToStruct_Block got wrong result, got:%#v", tc.val)
+			}
+		})
+	}
+}
+
 func TestImpliedBodySchema(t *testing.T) {
 	type testBlock struct {
 		Type       string `vcl:"type,label"`
@@ -378,6 +435,8 @@ func TestGetFieldTags(t *testing.T) {
 		Type     string      `vcl:"type,label"`
 		Name     string      `vcl:"name"` // implied attribute
 		Resource interface{} `vcl:"resource,block"`
+		Flats    interface{} `vcl:",flat"`
+		Comments interface{} `vcl:",comment"`
 	}
 
 	input := &testStruct{
@@ -408,5 +467,12 @@ func TestGetFieldTags(t *testing.T) {
 			t.Fatalf("Blocks length wrong[testCase:%d], got:%d, want:%d", n, len(tags.Blocks), 1)
 		}
 
+		if len(tags.Flats) != 1 {
+			t.Fatalf("Flats length wrong[testCase:%d], got:%d, want:%d", n, len(tags.Flats), 1)
+		}
+
+		if len(tags.Comments) != 1 {
+			t.Fatalf("Comments length wrong[testCase:%d], got:%d, want:%d", n, len(tags.Comments), 1)
+		}
 	}
 }

internal/lexer/lexer.go

@@ -74,6 +74,19 @@ func (l *Lexer) readPercentage(number string) string {
 	return number + "%"
 }
 
+func (l *Lexer) readCommentLine() string {
+	l.readChar()
+	pos := l.pos + 1 // Memo(KeisukeYamashita): Remove the first white space
+	for !isNewLine(l.char) {
+		l.readChar()
+		if l.char == 0 {
+			break
+		}
+	}
+
+	return l.input[pos:l.pos]
+}
+
 func (l *Lexer) peekChar() byte {
 	if l.readPos >= len(l.input) {
 		return 0
@@ -120,7 +133,26 @@ func (l *Lexer) NextToken() token.Token {
 	case ';':
 		tok = token.NewToken(token.SEMICOLON, l.char)
 	case '#':
-		tok = token.NewToken(token.COMMENT, l.char)
+		literal := l.readCommentLine()
+		tok = token.Token{Type: token.HASH, Literal: literal}
+	case '/':
+		if l.peekCharIs('/') {
+			l.readChar()
+			literal := l.readCommentLine()
+			tok = token.Token{Type: token.COMMENTLINE, Literal: literal}
+		} else if l.peekCharIs('*') {
+			char := l.char
+			l.readChar()
+			literal := string(char) + string(l.char)
+			tok = token.Token{Type: token.LMULTICOMMENTLINE, Literal: literal}
+		}
+	case '*':
+		if l.peekCharIs('/') {
+			char := l.char
+			l.readChar()
+			literal := string(char) + string(l.char)
+			tok = token.Token{Type: token.RMULTICOMMENTLINE, Literal: literal}
+		}
 	case '(':
 		tok = token.NewToken(token.LPAREN, l.char)
 	case ')':
@@ -193,3 +225,7 @@ func isLetter(char byte) bool {
 func isDigit(char byte) bool {
 	return '0' <= char && char <= '9'
 }
+
+func isNewLine(char byte) bool {
+	return char == '\n'
+}

internal/lexer/lexer_test.go

@@ -71,6 +71,9 @@ func TestNextToken(t *testing.T) {
 		},
 		{
 			`director my_dir random {
+				// keke
+				/* Hello */
+				# 3 hi
 				.retries = 3;
 			}`,
 			[]struct {
@@ -81,6 +84,11 @@ func TestNextToken(t *testing.T) {
 				{token.IDENT, "my_dir"},
 				{token.IDENT, "random"},
 				{token.LBRACE, "{"},
+				{token.COMMENTLINE, "keke"},
+				{token.LMULTICOMMENTLINE, "/*"},
+				{token.IDENT, "Hello"},
+				{token.RMULTICOMMENTLINE, "*/"},
+				{token.HASH, "3 hi"},
 				{token.IDENT, ".retries"},
 				{token.ASSIGN, "="},
 				{token.INT, "3"},

internal/parser/parser.go

@@ -308,6 +308,12 @@ func (p *Parser) parseStatement() ast.Statement {
 		default:
 			return p.parseExpressionStatement()
 		}
+	case token.LMULTICOMMENTLINE:
+		return p.parseMultiCommentStatement()
+	case token.HASH:
+		return p.parseCommentStatement()
+	case token.COMMENTLINE:
+		return p.parseCommentStatement()
 	case token.RETURN:
 		return p.parseReturnStatement()
 	case token.CALL:
@@ -399,6 +405,40 @@ func (p *Parser) parseReturnStatement() ast.Statement {
 	return stmt
 }
 
+func (p *Parser) parseCommentStatement() ast.Statement {
+	stmt := &ast.CommentStatement{
+		Token: p.curToken,
+		Value: p.curToken.Literal,
+	}
+
+	return stmt
+}
+
+func (p *Parser) parseMultiCommentStatement() ast.Statement {
+	stmt := &ast.CommentStatement{
+		Token: p.curToken,
+	}
+
+	var value string
+	p.nextToken()
+	for !p.curTokenIs(token.RMULTICOMMENTLINE) {
+		if p.curTokenIs(token.EOF) {
+			return nil
+		}
+
+		if value == "" {
+			value += p.curToken.Literal
+		} else {
+			value += " " + p.curToken.Literal
+		}
+
+		p.nextToken()
+	}
+
+	stmt.Value = value
+	return stmt
+}
+
 func (p *Parser) parseCallStatement() ast.Statement {
 	stmt := &ast.CallStatement{
 		Token: p.curToken,

internal/parser/parser_test.go

@@ -137,6 +137,43 @@ func testAssignStatement(t *testing.T, s ast.Statement, name string) bool {
 	return true
 }
 
+func TestCommentStatement(t *testing.T) {
+	testCases := map[string]struct {
+		input           string
+		expectedComment string
+	}{
+		"with comment line by hash":         {`# keke`, "keke"},
+		"with comment line by double slash": {"// keke", "keke"},
+		"with single comment by multi line": {"/* keke */", "keke"},
+		"with long comment by multi line":   {"/* keke is happy */", "keke is happy"},
+	}
+
+	for n, tc := range testCases {
+		t.Run(n, func(t *testing.T) {
+			l := lexer.NewLexer(tc.input)
+			p := NewParser(l)
+
+			program := p.ParseProgram()
+			if program == nil {
+				t.Fatalf("ParseProgram() failed got nil program")
+			}
+
+			if len(program.Statements) != 1 {
+				t.Fatalf("program.Statements wrong number returned, got:%d, want:%d", len(program.Statements), 1)
+			}
+
+			stmt, ok := program.Statements[0].(*ast.CommentStatement)
+			if !ok {
+				t.Fatalf("stmt was not ast.CommentStatement, got:%T", program.Statements[0])
+			}
+
+			if stmt.Value != tc.expectedComment {
+				t.Fatalf("stmt.Value got wrong value got:%s, want:%s", stmt.Value, tc.expectedComment)
+			}
+		})
+	}
+}
+
 func TestAssignFieldStatement(t *testing.T) {
 	testCases := []struct {
 		input               string