From 64f12762111f9385c50389e558cbb728388aa1af Mon Sep 17 00:00:00 2001 From: tompng Date: Thu, 30 May 2024 02:17:17 +0900 Subject: [PATCH] KeyStroke handles multibyte character --- lib/reline.rb | 3 ++- lib/reline/key_stroke.rb | 27 +++++++++++++++++++-------- lib/reline/line_editor.rb | 23 ++++------------------- test/reline/helper.rb | 8 +++----- test/reline/test_key_stroke.rb | 31 ++++++++++++++++++++++++++----- 5 files changed, 54 insertions(+), 38 deletions(-) diff --git a/lib/reline.rb b/lib/reline.rb index 720df286a1..7084cbaa70 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -308,6 +308,7 @@ def readline(prompt = '', add_hist = false) otio = io_gate.prep may_req_ambiguous_char_width + key_stroke.encoding = encoding line_editor.reset(prompt, encoding: encoding) if multiline line_editor.multiline_on @@ -483,7 +484,7 @@ def self.encoding_system_needs def self.core @core ||= Core.new { |core| core.config = Reline::Config.new - core.key_stroke = Reline::KeyStroke.new(core.config) + core.key_stroke = Reline::KeyStroke.new(core.config, core.encoding) core.line_editor = Reline::LineEditor.new(core.config, core.encoding) core.basic_word_break_characters = " \t\n`><=;|&{(" diff --git a/lib/reline/key_stroke.rb b/lib/reline/key_stroke.rb index ba40899685..e7b42a3687 100644 --- a/lib/reline/key_stroke.rb +++ b/lib/reline/key_stroke.rb @@ -3,8 +3,11 @@ class Reline::KeyStroke CSI_PARAMETER_BYTES_RANGE = 0x30..0x3f CSI_INTERMEDIATE_BYTES_RANGE = (0x20..0x2f) - def initialize(config) + attr_accessor :encoding + + def initialize(config, encoding) @config = config + @encoding = encoding end # Input exactly matches to a key sequence @@ -21,7 +24,7 @@ def match_status(input) matched = key_mapping.get(input) # FIXME: Workaround for single byte. remove this after MAPPING is merged into KeyActor. - matched ||= input.size == 1 + matched ||= input.size == 1 && input[0] < 0x80 matching ||= input == [ESC_BYTE] if matching && matched @@ -32,10 +35,14 @@ def match_status(input) MATCHED elsif input[0] == ESC_BYTE match_unknown_escape_sequence(input, vi_mode: @config.editing_mode_is?(:vi_insert, :vi_command)) - elsif input.size == 1 - MATCHED else - UNMATCHED + s = input.pack('c*').force_encoding(encoding) + if s.valid_encoding? + s.size == 1 ? MATCHED : UNMATCHED + else + # Invalid string is MATCHING (part of valid string) or MATCHED (invalid bytes to be ignored) + MATCHING_MATCHED + end end end @@ -45,6 +52,7 @@ def expand(input) bytes = input.take(i) status = match_status(bytes) matched_bytes = bytes if status == MATCHED || status == MATCHING_MATCHED + break if status == MATCHED || status == UNMATCHED end return [[], []] unless matched_bytes @@ -53,12 +61,15 @@ def expand(input) keys = func.map { |c| Reline::Key.new(c, c, false) } elsif func keys = [Reline::Key.new(func, func, false)] - elsif matched_bytes.size == 1 - keys = [Reline::Key.new(matched_bytes.first, matched_bytes.first, false)] elsif matched_bytes.size == 2 && matched_bytes[0] == ESC_BYTE keys = [Reline::Key.new(matched_bytes[1], matched_bytes[1] | 0b10000000, true)] else - keys = [] + s = matched_bytes.pack('c*').force_encoding(encoding) + if s.valid_encoding? && s.size == 1 + keys = [Reline::Key.new(s.ord, s.ord, false)] + else + keys = [] + end end [keys, input.drop(matched_bytes.size)] diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index d659378acd..78068af965 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -264,7 +264,6 @@ def reset_line @line_index = 0 @cache.clear @line_backup_in_history = nil - @multibyte_buffer = String.new(encoding: 'ASCII-8BIT') end def multiline_on @@ -1068,20 +1067,11 @@ def wrap_method_call(method_symbol, method_obj, key, with_operator = false) end private def normal_char(key) - @multibyte_buffer << key.combined_char - if @multibyte_buffer.size > 1 - if @multibyte_buffer.dup.force_encoding(@encoding).valid_encoding? - process_key(@multibyte_buffer.dup.force_encoding(@encoding), nil) - @multibyte_buffer.clear - else - # invalid - return - end - else # single byte - return if key.char >= 128 # maybe, first byte of multi byte + if key.char < 0x80 method_symbol = @config.editing_mode.get_method(key.combined_char) process_key(key.combined_char, method_symbol) - @multibyte_buffer.clear + else + process_key(key.char.chr(@encoding), nil) end if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize byte_size = Reline::Unicode.get_prev_mbchar_size(@buffer_of_lines[@line_index], @byte_pointer) @@ -1598,7 +1588,6 @@ def finish private def generate_searcher(search_key) search_word = String.new(encoding: @encoding) - multibyte_buf = String.new(encoding: 'ASCII-8BIT') hit_pointer = nil lambda do |key| search_again = false @@ -1613,11 +1602,7 @@ def finish search_again = true if search_key == key search_key = key else - multibyte_buf << key - if multibyte_buf.dup.force_encoding(@encoding).valid_encoding? - search_word << multibyte_buf.dup.force_encoding(@encoding) - multibyte_buf.clear - end + search_word << key end hit = nil if not search_word.empty? and @line_backup_in_history&.include?(search_word) diff --git a/test/reline/helper.rb b/test/reline/helper.rb index 9a346a17a1..5e3acc608f 100644 --- a/test/reline/helper.rb +++ b/test/reline/helper.rb @@ -121,17 +121,15 @@ def input_keys(input, convert = true) @line_editor.input_key(Reline::Key.new(byte, byte, false)) end else - c.bytes.each do |b| - @line_editor.input_key(Reline::Key.new(b, b, false)) - end + @line_editor.input_key(Reline::Key.new(c.ord, c.ord, false)) end end end def input_raw_keys(input, convert = true) input = convert_str(input) if convert - input.bytes.each do |b| - @line_editor.input_key(Reline::Key.new(b, b, false)) + input.chars.each do |c| + @line_editor.input_key(Reline::Key.new(c.ord, c.ord, false)) end end diff --git a/test/reline/test_key_stroke.rb b/test/reline/test_key_stroke.rb index ec70a05957..98080845dc 100644 --- a/test/reline/test_key_stroke.rb +++ b/test/reline/test_key_stroke.rb @@ -13,6 +13,10 @@ def to_keys end } + def encoding + Reline.core.encoding + end + def test_match_status config = Reline::Config.new { @@ -23,7 +27,7 @@ def test_match_status }.each_pair do |key, func| config.add_default_key_binding(key.bytes, func.bytes) end - stroke = Reline::KeyStroke.new(config) + stroke = Reline::KeyStroke.new(config, encoding) assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("a".bytes)) assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("ab".bytes)) assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("abc".bytes)) @@ -37,7 +41,7 @@ def test_match_status def test_match_unknown config = Reline::Config.new config.add_default_key_binding("\e[9abc".bytes, 'x') - stroke = Reline::KeyStroke.new(config) + stroke = Reline::KeyStroke.new(config, encoding) sequences = [ "\e[9abc", "\e[9d", @@ -66,7 +70,7 @@ def test_expand }.each_pair do |key, func| config.add_default_key_binding(key.bytes, func.bytes) end - stroke = Reline::KeyStroke.new(config) + stroke = Reline::KeyStroke.new(config, encoding) assert_equal(['123'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abcde'.bytes)) assert_equal(['456'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abde'.bytes)) # CSI sequence @@ -83,7 +87,7 @@ def test_oneshot_key_bindings }.each_pair do |key, func| config.add_default_key_binding(key.bytes, func.bytes) end - stroke = Reline::KeyStroke.new(config) + stroke = Reline::KeyStroke.new(config, encoding) assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('zzz'.bytes)) assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status('abc'.bytes)) end @@ -96,10 +100,27 @@ def test_with_reline_key }.each_pair do |key, func| config.add_oneshot_key_binding(key, func.bytes) end - stroke = Reline::KeyStroke.new(config) + stroke = Reline::KeyStroke.new(config, encoding) assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('da'.bytes)) assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("\eda".bytes)) assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status([32, 195, 164])) assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status([195, 164])) end + + def test_multibyte_matching + config = Reline::Config.new + stroke = Reline::KeyStroke.new(config, encoding) + char = 'あ'.encode(encoding) + key = Reline::Key.new(char.ord, char.ord, false) + bytes = char.bytes + assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status(bytes)) + assert_equal([[key], []], stroke.expand(bytes)) + assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status(bytes * 2)) + assert_equal([[key], bytes], stroke.expand(bytes * 2)) + (1...bytes.size).each do |i| + partial_bytes = bytes.take(i) + assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status(partial_bytes)) + assert_equal([[], []], stroke.expand(partial_bytes)) + end + end end