-
Notifications
You must be signed in to change notification settings - Fork 0
/
lexer.rb
89 lines (71 loc) · 1.57 KB
/
lexer.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# 引入 lib 下所有 *.rb文件
Dir[File.dirname(__FILE__) + '/lib/*.rb'].each {|file| require file }
class Lexer
attr_accessor :filename, :regex, :queue
RULES = [
[:ul, "(-\s)"],
[:blockquote, "(^>\s)"],
[:h6, "(^######\s)"],
[:h5, "(^#####\s)"],
[:h4, "(^####\s)"],
[:h3, "(^###\s)"],
[:h2, "(^##\s)"],
[:h1, "(^#\s)"],
[:code, "(^```)"],
[:text, "(.+)"]
]
RULE = RULES.collect {|k,v| v }.join('|')
def initialize(filename)
@filename = filename
@regex = Regexp.new(RULE)
@queue = []
read
end
def parser
Parser.new(@queue)
end
def read
File.readlines(@filename).each_with_index do |line, i|
# i 从 0 开始 需要先 + 1
@line_number = i + 1
readline(line)
end
end
private
def readline(line)
array_list = line.scan(@regex)
if array_list.length != 0
array_list.each do |array|
add_token(array)
end
else
# 空行
end
end
def add_token(array)
token = Token.new
array.each_with_index do |item, index|
if item != nil
token.type = RULES[index][0]
token.line_number = @line_number
if token.type == :text
token.text = item
end
# 找到单词 退出循环
break
end
end
if token.type.nil?
raise "bad token at line #{@line_number}"
end
@queue << token
end
end
# 读取文件 生成单词序列
lexer = Lexer.new("README.md")
# 读取单词序列 生成 AST
parser = lexer.parser
# 显示 AST
parser.show
# 转换为 HTML
puts parser.to_html