Skip to content

Commit

Permalink
KeyStroke handles multibyte character
Browse files Browse the repository at this point in the history
  • Loading branch information
tompng committed Jun 17, 2024
1 parent aff1d85 commit 64f1276
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 38 deletions.
3 changes: 2 additions & 1 deletion lib/reline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ def readline(prompt = '', add_hist = false)
otio = io_gate.prep

may_req_ambiguous_char_width
key_stroke.encoding = encoding
line_editor.reset(prompt, encoding: encoding)
if multiline
line_editor.multiline_on
Expand Down Expand Up @@ -483,7 +484,7 @@ def self.encoding_system_needs
def self.core
@core ||= Core.new { |core|
core.config = Reline::Config.new
core.key_stroke = Reline::KeyStroke.new(core.config)
core.key_stroke = Reline::KeyStroke.new(core.config, core.encoding)
core.line_editor = Reline::LineEditor.new(core.config, core.encoding)

core.basic_word_break_characters = " \t\n`><=;|&{("
Expand Down
27 changes: 19 additions & 8 deletions lib/reline/key_stroke.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ class Reline::KeyStroke
CSI_PARAMETER_BYTES_RANGE = 0x30..0x3f
CSI_INTERMEDIATE_BYTES_RANGE = (0x20..0x2f)

def initialize(config)
attr_accessor :encoding

def initialize(config, encoding)
@config = config
@encoding = encoding
end

# Input exactly matches to a key sequence
Expand All @@ -21,7 +24,7 @@ def match_status(input)
matched = key_mapping.get(input)

# FIXME: Workaround for single byte. remove this after MAPPING is merged into KeyActor.
matched ||= input.size == 1
matched ||= input.size == 1 && input[0] < 0x80
matching ||= input == [ESC_BYTE]

if matching && matched
Expand All @@ -32,10 +35,14 @@ def match_status(input)
MATCHED
elsif input[0] == ESC_BYTE
match_unknown_escape_sequence(input, vi_mode: @config.editing_mode_is?(:vi_insert, :vi_command))
elsif input.size == 1
MATCHED
else
UNMATCHED
s = input.pack('c*').force_encoding(encoding)
if s.valid_encoding?
s.size == 1 ? MATCHED : UNMATCHED
else
# Invalid string is MATCHING (part of valid string) or MATCHED (invalid bytes to be ignored)
MATCHING_MATCHED
end
end
end

Expand All @@ -45,6 +52,7 @@ def expand(input)
bytes = input.take(i)
status = match_status(bytes)
matched_bytes = bytes if status == MATCHED || status == MATCHING_MATCHED
break if status == MATCHED || status == UNMATCHED
end
return [[], []] unless matched_bytes

Expand All @@ -53,12 +61,15 @@ def expand(input)
keys = func.map { |c| Reline::Key.new(c, c, false) }
elsif func
keys = [Reline::Key.new(func, func, false)]
elsif matched_bytes.size == 1
keys = [Reline::Key.new(matched_bytes.first, matched_bytes.first, false)]
elsif matched_bytes.size == 2 && matched_bytes[0] == ESC_BYTE
keys = [Reline::Key.new(matched_bytes[1], matched_bytes[1] | 0b10000000, true)]
else
keys = []
s = matched_bytes.pack('c*').force_encoding(encoding)
if s.valid_encoding? && s.size == 1
keys = [Reline::Key.new(s.ord, s.ord, false)]
else
keys = []
end
end

[keys, input.drop(matched_bytes.size)]
Expand Down
23 changes: 4 additions & 19 deletions lib/reline/line_editor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,6 @@ def reset_line
@line_index = 0
@cache.clear
@line_backup_in_history = nil
@multibyte_buffer = String.new(encoding: 'ASCII-8BIT')
end

def multiline_on
Expand Down Expand Up @@ -1068,20 +1067,11 @@ def wrap_method_call(method_symbol, method_obj, key, with_operator = false)
end

private def normal_char(key)
@multibyte_buffer << key.combined_char
if @multibyte_buffer.size > 1
if @multibyte_buffer.dup.force_encoding(@encoding).valid_encoding?
process_key(@multibyte_buffer.dup.force_encoding(@encoding), nil)
@multibyte_buffer.clear
else
# invalid
return
end
else # single byte
return if key.char >= 128 # maybe, first byte of multi byte
if key.char < 0x80
method_symbol = @config.editing_mode.get_method(key.combined_char)
process_key(key.combined_char, method_symbol)
@multibyte_buffer.clear
else
process_key(key.char.chr(@encoding), nil)
end
if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize
byte_size = Reline::Unicode.get_prev_mbchar_size(@buffer_of_lines[@line_index], @byte_pointer)
Expand Down Expand Up @@ -1598,7 +1588,6 @@ def finish

private def generate_searcher(search_key)
search_word = String.new(encoding: @encoding)
multibyte_buf = String.new(encoding: 'ASCII-8BIT')
hit_pointer = nil
lambda do |key|
search_again = false
Expand All @@ -1613,11 +1602,7 @@ def finish
search_again = true if search_key == key
search_key = key
else
multibyte_buf << key
if multibyte_buf.dup.force_encoding(@encoding).valid_encoding?
search_word << multibyte_buf.dup.force_encoding(@encoding)
multibyte_buf.clear
end
search_word << key
end
hit = nil
if not search_word.empty? and @line_backup_in_history&.include?(search_word)
Expand Down
8 changes: 3 additions & 5 deletions test/reline/helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,15 @@ def input_keys(input, convert = true)
@line_editor.input_key(Reline::Key.new(byte, byte, false))
end
else
c.bytes.each do |b|
@line_editor.input_key(Reline::Key.new(b, b, false))
end
@line_editor.input_key(Reline::Key.new(c.ord, c.ord, false))
end
end
end

def input_raw_keys(input, convert = true)
input = convert_str(input) if convert
input.bytes.each do |b|
@line_editor.input_key(Reline::Key.new(b, b, false))
input.chars.each do |c|
@line_editor.input_key(Reline::Key.new(c.ord, c.ord, false))
end
end

Expand Down
31 changes: 26 additions & 5 deletions test/reline/test_key_stroke.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ def to_keys
end
}

def encoding
Reline.core.encoding
end

def test_match_status
config = Reline::Config.new
{
Expand All @@ -23,7 +27,7 @@ def test_match_status
}.each_pair do |key, func|
config.add_default_key_binding(key.bytes, func.bytes)
end
stroke = Reline::KeyStroke.new(config)
stroke = Reline::KeyStroke.new(config, encoding)
assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("a".bytes))
assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("ab".bytes))
assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("abc".bytes))
Expand All @@ -37,7 +41,7 @@ def test_match_status
def test_match_unknown
config = Reline::Config.new
config.add_default_key_binding("\e[9abc".bytes, 'x')
stroke = Reline::KeyStroke.new(config)
stroke = Reline::KeyStroke.new(config, encoding)
sequences = [
"\e[9abc",
"\e[9d",
Expand Down Expand Up @@ -66,7 +70,7 @@ def test_expand
}.each_pair do |key, func|
config.add_default_key_binding(key.bytes, func.bytes)
end
stroke = Reline::KeyStroke.new(config)
stroke = Reline::KeyStroke.new(config, encoding)
assert_equal(['123'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abcde'.bytes))
assert_equal(['456'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abde'.bytes))
# CSI sequence
Expand All @@ -83,7 +87,7 @@ def test_oneshot_key_bindings
}.each_pair do |key, func|
config.add_default_key_binding(key.bytes, func.bytes)
end
stroke = Reline::KeyStroke.new(config)
stroke = Reline::KeyStroke.new(config, encoding)
assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('zzz'.bytes))
assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status('abc'.bytes))
end
Expand All @@ -96,10 +100,27 @@ def test_with_reline_key
}.each_pair do |key, func|
config.add_oneshot_key_binding(key, func.bytes)
end
stroke = Reline::KeyStroke.new(config)
stroke = Reline::KeyStroke.new(config, encoding)
assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('da'.bytes))
assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("\eda".bytes))
assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status([32, 195, 164]))
assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status([195, 164]))
end

def test_multibyte_matching
config = Reline::Config.new
stroke = Reline::KeyStroke.new(config, encoding)
char = 'あ'.encode(encoding)
key = Reline::Key.new(char.ord, char.ord, false)
bytes = char.bytes
assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status(bytes))
assert_equal([[key], []], stroke.expand(bytes))
assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status(bytes * 2))
assert_equal([[key], bytes], stroke.expand(bytes * 2))
(1...bytes.size).each do |i|
partial_bytes = bytes.take(i)
assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status(partial_bytes))
assert_equal([[], []], stroke.expand(partial_bytes))
end
end
end

0 comments on commit 64f1276

Please sign in to comment.