From 37308fe3c0b2a6e6ebfe8ed0b7c4449e1f2ac68c Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Mon, 21 Nov 2022 10:03:48 -0500 Subject: [PATCH] Bring python yaml test parser of octet string inline with what javascript codegen is doing (#23670) * Fix octet string parsing in yaml test parser Old parsing had issue since str.encode for utf-8 character would be improperly convert. For example `\xff` would become `b'\xc3\xbf'` when we wanted it to be `b'\xff`. Co-authored-by: Tennessee Carmel-Veilleux * Address PR comments * run regen * Address PR comments Co-authored-by: Tennessee Carmel-Veilleux --- src/app/tests/suites/TestCluster.yaml | 3 +- .../python/chip/yaml/format_converter.py | 23 ++++++-- .../unit_tests/test_yaml_format_converter.py | 52 +++++++++++++++++++ .../chip-tool/zap-generated/test/Commands.h | 2 +- .../zap-generated/test/Commands.h | 2 +- 5 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 src/controller/python/test/unit_tests/test_yaml_format_converter.py diff --git a/src/app/tests/suites/TestCluster.yaml b/src/app/tests/suites/TestCluster.yaml index f84068e7ba912e..ddf4eb1362bc28 100644 --- a/src/app/tests/suites/TestCluster.yaml +++ b/src/app/tests/suites/TestCluster.yaml @@ -749,7 +749,8 @@ tests: command: "readAttribute" attribute: "octet_string" response: - value: "\r\n\xff\"\xa0" + # This is the properly 'hex:...' version of "\r\n\xff\"\xa0" + value: "hex:0d0aff22a0" - label: "Write attribute OCTET_STRING" command: "writeAttribute" diff --git a/src/controller/python/chip/yaml/format_converter.py b/src/controller/python/chip/yaml/format_converter.py index 11e38f8a9efb0a..f6402c59706058 100644 --- a/src/controller/python/chip/yaml/format_converter.py +++ b/src/controller/python/chip/yaml/format_converter.py @@ -20,9 +20,26 @@ from chip.tlv import uint, float32 import enum from chip.yaml.errors import ValidationError +import binascii -_HEX_PREFIX = 'hex:' +def convert_yaml_octet_string_to_bytes(s: str) -> bytes: + """Convert YAML octet string body to bytes, handling any c-style hex escapes (e.g. \x5a) and hex: prefix""" + # Step 1: handle explicit "hex:" prefix + if s.startswith('hex:'): + return binascii.unhexlify(s[4:]) + + # Step 2: convert non-hex-prefixed to bytes + # TODO(#23669): This does not properly support utf8 octet strings. We mimic + # javascript codegen behavior. Behavior of javascript is: + # * Octet string character >= u+0200 errors out. + # * Any character greater than 0xFF has the upper bytes chopped off. + as_bytes = [ord(c) for c in s] + + if any([value > 0x200 for value in as_bytes]): + raise ValueError('Unsupported char in octet string %r' % as_bytes) + accumulated_hex = ''.join([f"{(v & 0xFF):02x}" for v in as_bytes]) + return binascii.unhexlify(accumulated_hex) def convert_name_value_pair_to_dict(arg_values): @@ -118,9 +135,7 @@ def convert_yaml_type(field_value, field_type, use_from_dict=False): return field_type(field_value) # YAML treats bytes as strings. Convert to a byte string. elif (field_type == bytes and type(field_value) != bytes): - if isinstance(field_value, str) and field_value.startswith(_HEX_PREFIX): - return bytes.fromhex(field_value[len(_HEX_PREFIX):]) - return str.encode(field_value) + return convert_yaml_octet_string_to_bytes(field_value) # By default, just return the field_value casted to field_type. else: return field_type(field_value) diff --git a/src/controller/python/test/unit_tests/test_yaml_format_converter.py b/src/controller/python/test/unit_tests/test_yaml_format_converter.py new file mode 100644 index 00000000000000..fa46705be65ed9 --- /dev/null +++ b/src/controller/python/test/unit_tests/test_yaml_format_converter.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from chip.yaml.format_converter import convert_yaml_octet_string_to_bytes +from binascii import unhexlify +import unittest + + +class TestOctetStringYamlDecode(unittest.TestCase): + def test_common_cases(self): + self.assertEqual(convert_yaml_octet_string_to_bytes("hex:aa55"), unhexlify("aa55")) + self.assertEqual(convert_yaml_octet_string_to_bytes("hex:"), unhexlify("")) + self.assertEqual(convert_yaml_octet_string_to_bytes("hex:AA55"), unhexlify("aa55")) + + self.assertEqual(convert_yaml_octet_string_to_bytes("0\xaa\x55"), unhexlify("30aa55")) + self.assertEqual(convert_yaml_octet_string_to_bytes("0\xAA\x55"), unhexlify("30aa55")) + self.assertEqual(convert_yaml_octet_string_to_bytes("0\xAa\x55"), unhexlify("30aa55")) + + self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:"), b"0hex:") + self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:A"), b"0hex:A") + self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:AA55"), b"0hex:AA55") + + self.assertEqual(convert_yaml_octet_string_to_bytes("AA55"), b"AA55") + self.assertEqual(convert_yaml_octet_string_to_bytes("AA\n\r\t55"), unhexlify("41410a0d093535")) + # TODO(#23669): After utf8 is properly supported expected result is unhexlify("c3a9c3a90a0a") + self.assertEqual(convert_yaml_octet_string_to_bytes("\xC3\xA9é\n\n"), unhexlify("c3a9e90a0a")) + + # Partial hex nibble + with self.assertRaises(ValueError): + convert_yaml_octet_string_to_bytes("hex:aa5") + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/zzz_generated/chip-tool/zap-generated/test/Commands.h b/zzz_generated/chip-tool/zap-generated/test/Commands.h index e9f193028de534..796a968a9bbed7 100644 --- a/zzz_generated/chip-tool/zap-generated/test/Commands.h +++ b/zzz_generated/chip-tool/zap-generated/test/Commands.h @@ -48621,7 +48621,7 @@ class TestClusterSuite : public TestCommand chip::ByteSpan value; VerifyOrReturn(CheckDecodeValue(chip::app::DataModel::Decode(*data, value))); VerifyOrReturn(CheckValueAsString("octetString", value, - chip::ByteSpan(chip::Uint8::from_const_char("\015\012\377\042\240"), 5))); + chip::ByteSpan(chip::Uint8::from_const_char("\x0d\x0a\xff\x22\xa0"), 5))); } break; case 114: diff --git a/zzz_generated/darwin-framework-tool/zap-generated/test/Commands.h b/zzz_generated/darwin-framework-tool/zap-generated/test/Commands.h index 3fb1a5cc8884a1..2b90c663601ca8 100644 --- a/zzz_generated/darwin-framework-tool/zap-generated/test/Commands.h +++ b/zzz_generated/darwin-framework-tool/zap-generated/test/Commands.h @@ -77591,7 +77591,7 @@ class TestCluster : public TestCommandBridge { { id actualValue = value; VerifyOrReturn(CheckValueAsString( - "octet_string", actualValue, [[NSData alloc] initWithBytes:"\015\012\377\042\240" length:5])); + "octet_string", actualValue, [[NSData alloc] initWithBytes:"\x0d\x0a\xff\x22\xa0" length:5])); } NextTest();