From 26dae8d94a2cd959e407bf3258b0a81c6e1331d2 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 Feb 2023 14:10:41 -0800 Subject: [PATCH 001/463] [ObjC] Test pddm expansion Testing both the support and the sources expanded. PiperOrigin-RevId: 508471876 --- objectivec/BUILD.bazel | 28 +++++++++++++++++++ .../DevTools/sources_pddm_expansion_test.sh | 9 ++++++ 2 files changed, 37 insertions(+) create mode 100755 objectivec/DevTools/sources_pddm_expansion_test.sh diff --git a/objectivec/BUILD.bazel b/objectivec/BUILD.bazel index 1c453574629d..3a63685b2f58 100644 --- a/objectivec/BUILD.bazel +++ b/objectivec/BUILD.bazel @@ -121,6 +121,34 @@ sh_test( ], ) +# ------------------------------------------------------------------- +# Validation of pddm expansion. + +py_binary( + name = "pddm", + srcs = ["DevTools/pddm.py"], +) + +py_test( + name = "pddm_tests", + size = "small", + srcs = [ + "DevTools/pddm.py", + "DevTools/pddm_tests.py", + ], +) + +sh_test( + name = "sources_pddm_expansion_test", + size = "small", + srcs = ["DevTools/sources_pddm_expansion_test.sh"], + data = [":pddm"] + glob([ + "**/*.h", + "**/*.m", + "**/*.pddm", + ]), +) + ################################################################################ # Distribution files ################################################################################ diff --git a/objectivec/DevTools/sources_pddm_expansion_test.sh b/objectivec/DevTools/sources_pddm_expansion_test.sh new file mode 100755 index 000000000000..99e01a1bcd6f --- /dev/null +++ b/objectivec/DevTools/sources_pddm_expansion_test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +${TEST_SRCDIR}/google3/third_party/protobuf/objectivec/pddm \ + --dry-run \ + ${TEST_SRCDIR}/google3/third_party/protobuf/objectivec/*.[hm] \ + ${TEST_SRCDIR}/google3/third_party/protobuf/objectivec/Tests/*.[hm] \ + || die "Update by running: objectivec/DevTools/pddm.py objectivec/*.[hm] objectivec/Tests/*.[hm]" + +echo "PASS" From 23d8aacc551bb10694e6b32a74c885b64d9e1501 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 Feb 2023 15:00:37 -0800 Subject: [PATCH 002/463] Updates links to use the new protobuf.dev site, and adds a link to the new version support page. PiperOrigin-RevId: 508485038 --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3bfeb0f16901..c84ab2b75a1b 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,12 @@ Protocol Buffers - Google's data interchange format Copyright 2008 Google Inc. -[Protocol Buffers documentation](https://developers.google.com/protocol-buffers/) - Overview -------- Protocol Buffers (a.k.a., protobuf) are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data. You -can find [protobuf's documentation on the Google Developers site](https://developers.google.com/protocol-buffers/). +can learn more about it in [protobuf's documentation](https://protobuf.dev). This README file contains protobuf installation instructions. To install protobuf, you need to install the protocol compiler (used to compile .proto @@ -64,7 +62,7 @@ Quick Start ----------- The best way to learn how to use protobuf is to follow the [tutorials in our -developer guide](https://developers.google.com/protocol-buffers/docs/tutorials). +developer guide](https://protobuf.dev/getting-started). If you want to learn from code examples, take a look at the examples in the [examples](examples) directory. @@ -72,11 +70,16 @@ If you want to learn from code examples, take a look at the examples in the Documentation ------------- -The complete documentation is available via the [Protocol Buffers documentation](https://developers.google.com/protocol-buffers/). +The complete documentation is available at the [Protocol Buffers doc site](https://protobuf.dev). + +Support Policy +-------------- + +Read about our [version support policy](https://protobuf.dev/version-support/) +to stay current on support timeframes for the language libraries. Developer Community ------------------- To be alerted to upcoming changes in Protocol Buffers and connect with protobuf developers and users, [join the Google Group](https://groups.google.com/g/protobuf). - From 2ef1b7a9fad4c02c7e85aeac4819fb44406b2a17 Mon Sep 17 00:00:00 2001 From: Daniel Azuma Date: Thu, 9 Feb 2023 20:20:15 -0800 Subject: [PATCH 003/463] fix(ruby): Fixed include paths for Ruby Windows builds (#11882) This fixes a v22 regression for Ruby. Windows builds of the Ruby package are broken in 3.22.0.rc.2. The reason is, `ruby-upb.h` no longer provides the full path when `#include`ing `utf8_range.h`. See https://github.com/protocolbuffers/protobuf/blob/v22.0-rc2/ruby/ext/google/protobuf_c/ruby-upb.h#L10479 (22.0-rc-2), cf. https://github.com/protocolbuffers/protobuf/blob/v21.12/ruby/ext/google/protobuf_c/ruby-upb.h#L5365 (21.12). The `extconf.rb` build configuration tries to compensate by adding `third_party/utf8_range` to the include path, but does not do so on Windows (i.e. `mingw`) platforms. See https://github.com/protocolbuffers/protobuf/blob/v22.0-rc2/ruby/ext/google/protobuf_c/extconf.rb#L9-L10 (22.0-rc-2), cf. https://github.com/protocolbuffers/protobuf/blob/v21.12/ruby/ext/google/protobuf_c/extconf.rb#L9-L10 (21.12). We could have fixed this by adding another clause to the if statement for the case `RUBY_PLATFORM =~ /mingw/` and adding the appropriate `-I` flag to `CFLAGS`. However, that `CFLAGS` hack is present in the first place due to a related problem: the usage of `$INCFLAGS` is incorrect. The `$INCFLAGS` constant in Ruby's `mkmf` is a string in a similar format to `CFLAGS`. It's simply appended to compiler invocations. So when you append new flags to it, you have to (1) provide the flag itself, and (2) precede it with a space to delimit it from the previous entry. In 22.0-rc-2 (and indeed in all earlier versions) the usage is incorrect: it's appending a path to the string without the `-I` and without a space. See https://github.com/protocolbuffers/protobuf/blob/v22.0-rc2/ruby/ext/google/protobuf_c/extconf.rb#L22. Hence, not only does the intended include path not get appended correctly, it also clobbers the previous path in the string. Luckily, the previous path is only `-I$(srcdir)` which happens not to matter for our library. But it does mean that the apparent intent of that line, adding `$(srcdir)/third_party/utf8_range` to the include path, isn't working; hence the code that adds it to `CFLAGS` instead. (Note that the previous line, adding the path to `$VPATH`, _is_ correct as is, because `$VPATH` is an array.) So what this PR actually does is fix the `$INCFLAGS` usage so `$(srcdir)/third_party/utf8_range` gets added properly to compiler include paths, for all platforms including Windows. Since that should now be working as intended, we also remove the extra `-I` from CFLAGS. Builds for all platforms should now be able to handle the change to `ruby-upb.h`. This has been tested by running the prototype Ruby build Kokoro job against a patched 22.0-rc-2. This also needs to be backported to the 22.x branch. /attn @deannagarcia Closes #11882 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11882 from dazuma:pr/ruby-builds-fix ebb18e000479694ad423165378ddcb2df3c6001c PiperOrigin-RevId: 508550039 --- ruby/ext/google/protobuf_c/extconf.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ruby/ext/google/protobuf_c/extconf.rb b/ruby/ext/google/protobuf_c/extconf.rb index 04c6f78fd3f9..b7c439b0b2a6 100755 --- a/ruby/ext/google/protobuf_c/extconf.rb +++ b/ruby/ext/google/protobuf_c/extconf.rb @@ -7,19 +7,18 @@ dir_config(ext_name) if RUBY_PLATFORM =~ /darwin/ || RUBY_PLATFORM =~ /linux/ || RUBY_PLATFORM =~ /freebsd/ - $CFLAGS += " -std=gnu99 -O3 -DNDEBUG -fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement -I$(srcdir)/third_party/utf8_range" + $CFLAGS += " -std=gnu99 -O3 -DNDEBUG -fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" else $CFLAGS += " -std=gnu99 -O3 -DNDEBUG" end - if RUBY_PLATFORM =~ /linux/ # Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper. $LDFLAGS += " -Wl,-wrap,memcpy" end $VPATH << "$(srcdir)/third_party/utf8_range" -$INCFLAGS << "$(srcdir)/third_party/utf8_range" +$INCFLAGS += " -I$(srcdir)/third_party/utf8_range" $srcs = ["protobuf.c", "convert.c", "defs.c", "message.c", "repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c", From a50e49530f4e837f63fdbb2cb58208fcb4d67936 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 08:48:09 -0800 Subject: [PATCH 004/463] Always enable eagerly verified lazy fields. PiperOrigin-RevId: 508671173 --- src/google/protobuf/compiler/cpp/helpers.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index e815e8b71d2d..7b1414955e60 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -211,9 +211,9 @@ bool IsLazy(const FieldDescriptor* field, const Options& options, // Returns true if "field" is a message field that is backed by LazyField per // profile (go/pdlazy). -inline bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field, - const Options& options, - MessageSCCAnalyzer* scc_analyzer) { +inline bool IsLazyByProfile(const FieldDescriptor* field, + const Options& options, + MessageSCCAnalyzer* scc_analyzer) { return false; } From 76573bfef5c4c2a25bc1ecd2bb139e6b70a2ebf6 Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Fri, 10 Feb 2023 09:08:30 -0800 Subject: [PATCH 005/463] Remove data_offset + hasbit_idx template vars PiperOrigin-RevId: 508675626 --- .../protobuf/generated_message_tctable_impl.h | 109 +++--------------- .../generated_message_tctable_lite.cc | 29 ++++- .../generated_message_tctable_lite_test.cc | 7 +- 3 files changed, 51 insertions(+), 94 deletions(-) diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index a39245db5f70..4f2cc2a51619 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -360,35 +360,21 @@ class PROTOBUF_EXPORT TcParser final { static const char* FastZ64P2(PROTOBUF_TC_PARAM_DECL); // Manually unrolled and specialized Varint parsing. - template + template static const char* FastTV32S1(PROTOBUF_TC_PARAM_DECL); - template + template static const char* FastTV64S1(PROTOBUF_TC_PARAM_DECL); - template - static const char* FastTV8S1(PROTOBUF_TC_PARAM_DECL); - template + template static constexpr TailCallParseFunc SingularVarintNoZag1() { if (sizeof(FieldType) == 1) { - if (data_offset < 100) { - return &FastTV8S1; - } else { - return &FastV8S1; - } + return &FastV8S1; } if (sizeof(FieldType) == 4) { - if (data_offset < 100) { - return &FastTV32S1; - } else { // - return &FastV32S1; - } + return &FastTV32S1; } if (sizeof(FieldType) == 8) { - if (data_offset < 128) { - return &FastTV64S1; - } else { - return &FastV64S1; - } + return &FastTV64S1; } static_assert(sizeof(FieldType) == 1 || sizeof(FieldType) == 4 || sizeof(FieldType) == 8, @@ -856,67 +842,15 @@ ParseFallbackPair(const char* p, int64_t res1) { return {p + 10, res1 & res2 & res3}; } -// Notes: -// 1) if data_offset is negative, it's read from data.offset() -// 2) if hasbit_idx is negative, it's read from data.hasbit_idx() -template -PROTOBUF_NOINLINE const char* TcParser::FastTV8S1(PROTOBUF_TC_PARAM_DECL) { - using TagType = uint8_t; - - // Special case for a varint bool field with a tag of 1 byte: - // The coded_tag() field will actually contain the value too and we can check - // both at the same time. - auto coded_tag = data.coded_tag(); - if (PROTOBUF_PREDICT_TRUE(coded_tag == 0x0000 || coded_tag == 0x0100)) { - auto& field = - RefAt(msg, data_offset >= 0 ? data_offset : data.offset()); - // Note: we use `data.data` because Clang generates suboptimal code when - // using coded_tag. - // In x86_64 this uses the CH register to read the second byte out of - // `data`. - uint8_t value = data.data >> 8; - // The assume allows using a mov instead of test+setne. - PROTOBUF_ASSUME(value <= 1); - field = static_cast(value); - - ptr += sizeof(TagType) + 1; // Consume the tag and the value. - if (hasbit_idx < 0) { - hasbits |= (uint64_t{1} << data.hasbit_idx()); - } else { - if (hasbit_idx < 32) { - // `& 31` avoids a compiler warning when hasbit_idx is negative. - hasbits |= (uint64_t{1} << (hasbit_idx & 31)); - } else { - static_assert(hasbit_idx == 63 || (hasbit_idx < 32), - "hard-coded hasbit_idx should be 0-31, or the special" - "value 63, which indicates the field has no has-bit."); - // TODO(jorg): investigate whether higher hasbit indices are worth - // supporting. Something like: - // auto& hasblock = TcParser::RefAt(msg, hasbit_idx / 32 * 4); - // hasblock |= uint32_t{1} << (hasbit_idx % 32); - } - } - - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); - } - - // If it didn't match above either the tag is wrong, or the value is encoded - // non-canonically. - // Jump to MiniParse as wrong tag is the most probable reason. - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); -} - -template +template PROTOBUF_NOINLINE const char* TcParser::FastTV64S1(PROTOBUF_TC_PARAM_DECL) { using TagType = uint8_t; // super-early success test... if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { ptr += sizeof(TagType); // Consume tag - if (hasbit_idx < 32) { - hasbits |= (uint64_t{1} << hasbit_idx); - } + hasbits |= (uint64_t{1} << data.hasbit_idx()); uint8_t value = data.data >> 8; - RefAt(msg, data_offset) = value; + RefAt(msg, data.offset()) = value; ptr += 1; PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } @@ -924,33 +858,30 @@ PROTOBUF_NOINLINE const char* TcParser::FastTV64S1(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } ptr += sizeof(TagType); // Consume tag - if (hasbit_idx < 32) { - hasbits |= (uint64_t{1} << hasbit_idx); - } + hasbits |= (uint64_t{1} << data.hasbit_idx()); auto tmp = ParseFallbackPair(ptr, static_cast(data.data >> 8)); - data.data = 0; // Indicate to the compiler that we don't need this anymore. ptr = tmp.first; if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + data.data = 0; // Indicate to the compiler that we don't need this anymore. return Error(PROTOBUF_TC_PARAM_PASS); } - RefAt(msg, data_offset) = static_cast(tmp.second); + RefAt(msg, data.offset()) = static_cast(tmp.second); + data.data = 0; // Indicate to the compiler that we don't need this anymore. PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } -template +template PROTOBUF_NOINLINE const char* TcParser::FastTV32S1(PROTOBUF_TC_PARAM_DECL) { using TagType = uint8_t; // super-early success test... if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { ptr += sizeof(TagType); // Consume tag - if (hasbit_idx < 32) { - hasbits |= (uint64_t{1} << hasbit_idx); - } + hasbits |= (uint64_t{1} << data.hasbit_idx()); uint8_t value = data.data >> 8; - RefAt(msg, data_offset) = value; + RefAt(msg, data.offset()) = value; ptr += 1; PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } @@ -958,19 +889,17 @@ PROTOBUF_NOINLINE const char* TcParser::FastTV32S1(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } ptr += sizeof(TagType); // Consume tag - if (hasbit_idx < 32) { - hasbits |= (uint64_t{1} << hasbit_idx); - } + hasbits |= (uint64_t{1} << data.hasbit_idx()); auto tmp = ParseFallbackPair(ptr, static_cast(data.data >> 8)); - data.data = 0; // Indicate to the compiler that we don't need this anymore. ptr = tmp.first; if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { return Error(PROTOBUF_TC_PARAM_PASS); } - RefAt(msg, data_offset) = static_cast(tmp.second); + RefAt(msg, data.offset()) = static_cast(tmp.second); + data.data = 0; // Indicate to the compiler that we don't need this anymore. PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 86d067b01075..acc1915efa8e 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -850,8 +850,35 @@ PROTOBUF_NOINLINE const char* TcParser::SingularVarBigint( } PROTOBUF_NOINLINE const char* TcParser::FastV8S1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return FastTV8S1<-1, -1>(PROTOBUF_TC_PARAM_PASS); + using TagType = uint8_t; + + // Special case for a varint bool field with a tag of 1 byte: + // The coded_tag() field will actually contain the value too and we can check + // both at the same time. + auto coded_tag = data.coded_tag(); + if (PROTOBUF_PREDICT_TRUE(coded_tag == 0x0000 || coded_tag == 0x0100)) { + auto& field = RefAt(msg, data.offset()); + // Note: we use `data.data` because Clang generates suboptimal code when + // using coded_tag. + // In x86_64 this uses the CH register to read the second byte out of + // `data`. + uint8_t value = data.data >> 8; + // The assume allows using a mov instead of test+setne. + PROTOBUF_ASSUME(value <= 1); + field = static_cast(value); + + ptr += sizeof(TagType) + 1; // Consume the tag and the value. + hasbits |= (uint64_t{1} << data.hasbit_idx()); + + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + + // If it didn't match above either the tag is wrong, or the value is encoded + // non-canonically. + // Jump to MiniParse as wrong tag is the most probable reason. + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } + PROTOBUF_NOINLINE const char* TcParser::FastV8S2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return SingularVarint( PROTOBUF_TC_PARAM_PASS); diff --git a/src/google/protobuf/generated_message_tctable_lite_test.cc b/src/google/protobuf/generated_message_tctable_lite_test.cc index e242d839eed5..0d923aeb4a27 100644 --- a/src/google/protobuf/generated_message_tctable_lite_test.cc +++ b/src/google/protobuf/generated_message_tctable_lite_test.cc @@ -125,6 +125,7 @@ TEST(FastVarints, NameHere) { // clang-format on uint8_t serialize_buffer[64]; + // TODO(b/27721823): cleanup test cases for 'former' TV* functions. for (int size : {8, 32, 64, -8, -32, -64}) { SCOPED_TRACE(size); auto next_i = [](uint64_t i) { @@ -193,19 +194,19 @@ TEST(FastVarints, NameHere) { fn = &TcParser::FastV8S1; break; case -8: - fn = &TcParser::FastTV8S1; + fn = &TcParser::FastV8S1; break; case 32: fn = &TcParser::FastV32S1; break; case -32: - fn = &TcParser::FastTV32S1; + fn = &TcParser::FastTV32S1; break; case 64: fn = &TcParser::FastV64S1; break; case -64: - fn = &TcParser::FastTV64S1; + fn = &TcParser::FastTV64S1; break; } fallback_ptr_received = absl::nullopt; From 6aefc477d74babfc093a5721510af9a0ec12a790 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 09:30:14 -0800 Subject: [PATCH 006/463] Add documentation for our new GHA infrastructure PiperOrigin-RevId: 508680514 --- .github/README.md | 196 ++++++++++++++++++++++++++++++++++++++++++++++ ci/README.md | 17 ++++ 2 files changed, 213 insertions(+) create mode 100644 .github/README.md create mode 100644 ci/README.md diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 000000000000..b1c780323c79 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,196 @@ +This directory contains all of our automatically triggered workflows. + +# Test runner + +Our top level `test_runner.yml` is responsible for kicking off all tests, which +are represented as reusable workflows. This is carefully constructed to satisfy +the design laid out in go/protobuf-gha-protected-resources (see below), and +duplicating it across every workflow file would be difficult to maintain. As an +added bonus, we can manually dispatch our full test suite with a single button +and monitor the progress of all of them simultaneously in GitHub's actions UI. + +There are five ways our test suite can be triggered: + +- **Post-submit tests** (`push`): These are run over newly submitted code +that we can assume has been thoroughly reviewed. There are no additional +security concerns here and these jobs can be given highly privileged access to +our internal resources and caches. + +- **Pre-submit tests from a branch** (`push_request`): These are run over +every PR as changes are made. Since they are coming from branches in our +repository, they have secret access by default and can also be given highly +privileged access. However, we expect *many* of these events per change, +and likely many from abandoned/exploratory changes. Given the much higher +frequency, we restrict the ability to *write* to our more expensive caches. + +- **Pre-submit tests from a fork** (`push_request_target`): These are run +over every PR from a forked repository as changes are made. These have much +more restricted access, since they could be coming from anywhere. To protect +our secret keys and our resources, tests will not run until a commit has been +labeled `safe to submit`. Further commits will require further approvals to +run our test suite. Once marked as safe, we will provide read-only access to +our caches and Docker images, but will generally disallow any writes to shared +resources. + +- **Continuous tests** (`schedule`): These are run on a fixed schedule. We +currently have them set up to run daily, and can help identify non-hermetic +issues in tests that don't get run often (such as due to test caching) or during +slow periods like weekends and holidays. Similar to post-submit tests, these +are run over submitted code and are highly privileged in the resources they +can use. + +- **Manual testing** (`workflow_dispatch`): Our test runner can be triggered +manually over any branch. This is treated similarly to pre-submit tests, +which should be highly privileged because they can only be triggered by the +protobuf team. + +# Staleness handling + +While Bazel handles code generation seamlessly, we do support build systems that +don't. There are a handful of cases where we need to check in generated files +that can become stale over time. In order to provide a good developer +experience, we've implemented a system to make this more manageable. + +- Stale files should have a corresponding `staleness_test` Bazel target. This +should be marked `manual` to avoid getting picked up in CI, but will fail if +files become stale. It also provides a `--fix` flag to update the stale files. + +- Bazel tests will never depend on the checked-in versions, and will generate +new ones on-the-fly during build. + +- Non-Bazel tests will always regenerate necessary files before starting. This +is done using our `bash` and `docker` actions, which should be used for any +non-Bazel tests. This way, no tests will fail due to stale files. + +- A post-submit job will immediately regenerate any stale files and commit them +if they've changed. + +- A scheduled job will run late at night every day to make sure the post-submit +is working as expected (that is, it will run all the staleness tests). + +The `regenerate_stale_files.sh` script is the central script responsible for all +the re-generation of stale files. + +# Caches + +We have a number of different caching strategies to help speed up tests. These +live either in GCP buckets or in our GitHub repository cache. The former has +a lot of resources available and we don't have to worry as much about bloat. +On the other hand, the GitHub repository cache is limited to 10GB, and will +start pruning old caches when it exceeds that threshold. Therefore, we need +to be very careful about the size and quantity of our caches in order to +maximize the gains. + +## Bazel remote cache + +As described in https://bazel.build/remote/caching, remote caching allows us to +offload a lot of our build steps to a remote server that holds a cache of +previous builds. We use our GCP project for this storage, and configure +*every* Bazel call to use it. This provides substantial performance +improvements at minimal cost. + +We do not allow forked PRs to upload updates to our Bazel caches, but they +do use them. Every other event is given read/write access to the caches. +Because Bazel behaves poorly under certain environment changes (such as +toolchain, operating system), we try to use finely-grained caches. Each job +should typically have its own cache to avoid cross-pollution. + +## Bazel repository cache + +When Bazel starts up, it downloads all the external dependencies for a given +build and stores them in the repository cache. This cache is *separate* from +the remote cache, and only exists locally. Because we have so many Bazel +dependencies, this can be a source of frequent flakes due to network issues. + +To avoid this, we keep a cached version of the repository cache in GitHub's +action cache. Our full set of repository dependencies ends up being ~300MB, +which is fairly expensive given our 10GB maximum. The most expensive ones seem +to come from Java, which has some very large downstream dependencies. + +Given the cost, we take a more conservative approach for this cache. Only push +events will ever write to this cache, but all events can read from them. +Additionally, we only store three caches for any given commit, one per platform. +This means that multiple jobs are trying to update the same cache, leading to a +race. GitHub rejects all but one of these updates, so we designed the system so +that caches are only updated if they've actually changed. That way, over time +(and multiple pushes) the repository caches will incrementally grow to encompass +all of our dependencies. A scheduled job will run monthly to clear these caches +to prevent unbounded growth as our dependencies evolve. + +## ccache + +In order to speed up non-Bazel builds to be on par with Bazel, we make use of +[ccache](https://ccache.dev/). This intercepts all calls to the compiler, and +caches the result. Subsequent calls with a cache-hit will very quickly +short-circuit and return the already computed result. This has minimal affect +on any *single* job, since we typically only run a single build. However, by +caching the ccache results in GitHub's action cache we can substantially +decrease the build time of subsequent runs. + +One useful feature of ccache is that you can set a maximum cache size, and it +will automatically prune older results to keep below that limit. On Linux and +Mac cmake builds, we generally get 30MB caches and set a 100MB cache limit. On +Windows, with debug symbol stripping we get ~70MB and set a 200MB cache limit. + +Because CMake build tend to be our slowest, bottlenecking the entire CI process, +we use a fairly expensive strategy with ccache. All events will cache their +ccache directory, keyed by the commit and the branch. This means that each +PR and each branch will write its own set of caches. When looking up which +cache to use initially, each job will first look for a recent cache in its +current branch. If it can't find one, it will accept a cache from the base +branch (for example, PRs will initially use the latest cache from their target +branch). + +While the ccache caches quickly over-run our GitHub action cache, they also +quickly become useless. Since GitHub prunes caches based on the time they were +last used, this just means that we'll see quicker turnover. + +## Bazelisk + +Bazelisk will automatically download a pinned version of Bazel on first use. +This can lead to flakes, and to avoid that we cache the result keyed on the +Bazel version. Only push events will write to this cache, but it's unlikely +to change very often. + +## Docker images + +Instead of downloading a fresh Docker image for every test run, we can save it +as a tar and cache it using `docker image save` and later restore using +`docker image load`. This can decrease download times and also reduce flakes. +Note, Docker's load can actually be significantly slower than a pull in certain +situations. Therefore, we should reserve this strategy for only Docker images +that are causing noticeable flakes. + +## Pip dependencies + +The actions/setup-python action we use for Python supports automated caching +of pip dependencies. We enable this to avoid having to download these +dependencies on every run, which can lead to flakes. + +# Custom actions + +We've defined a number of custom actions to abstract out shared pieces of our +workflows. + +- **Bazel** use this for running all Bazel tests. It can take either a single +Bazel command or a more general bash command. In the latter case, it provides +environment variables for running Bazel with all our standardized settings. + +- **Bazel-Docker** nearly identical to the **Bazel** action, this additionally +runs everything in a specified Docker image. + +- **Bash** use this for running non-Bazel tests. It takes a bash command and +runs it verbatim. It also handles the regeneration of stale files (which does +use Bazel), which non-Bazel tests might depend on. + +- **Docker** nearly identical to the **Bash** action, this additionally runs +everything in a specified Docker image. + +- **ccache** this sets up a ccache environment, and initializes some +environment variables for standardized usage of ccache. + +- **Cross-compile protoc** this abstracts out the compilation of protoc using +our cross-compilation infrastructure. It will set a `PROTOC` environment +variable that gets automatically picked up by a lot of our infrastructure. +This is most useful in conjunction with the **Bash** action with non-Bazel +tests. diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000000..01c8373145c8 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,17 @@ +This directory contains CI-specific tooling. + +# Clang wrappers + +CMake allows for compiler wrappers to be injected such as ccache, which +intercepts compiler calls and short-circuits on cache-hits. This can be done +by specifying `CMAKE_C_COMPILER_LAUNCHER` and `CMAKE_CXX_COMPILER_LAUNCHER` +during CMake's configure step. Unfortunately, X-Code doesn't provide anything +like this, so we use basic wrapper scripts to invoke ccache + clang. + +# Bazelrc files + +In order to allow platform-specific `.bazelrc` flags during testing, we keep +3 different versions here along with a shared `common.bazelrc` that they all +include. Our GHA infrastructure will select the appropriate file for any test +and overwrite the default `.bazelrc` in our workspace, which is intended for +development only. From 363fa89b1f02d4c51028f8c8bcd506f08eaa6b49 Mon Sep 17 00:00:00 2001 From: Andrei Tcaci Date: Fri, 10 Feb 2023 10:16:28 -0800 Subject: [PATCH 007/463] RepeatedField: unset by index (#11425) Instead of using `array_pop()` to remove last element use `unset()` to remove by index Closes #11425 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11425 from Leprechaunz:unset-by-index a47fba57e43228c92791aa75e1bc04169491bf41 PiperOrigin-RevId: 508691545 --- php/ext/google/protobuf/array.c | 4 +-- .../Protobuf/Internal/RepeatedField.php | 4 +-- php/tests/ArrayTest.php | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/php/ext/google/protobuf/array.c b/php/ext/google/protobuf/array.c index 72c7809674c6..09daffa16129 100644 --- a/php/ext/google/protobuf/array.c +++ b/php/ext/google/protobuf/array.c @@ -406,13 +406,13 @@ PHP_METHOD(RepeatedField, offsetUnset) { return; } - if (size == 0 || index != size - 1) { + if (size == 0 || index < 0 || index >= size) { php_error_docref(NULL, E_USER_ERROR, "Cannot remove element at %ld.\n", index); return; } - upb_Array_Resize(intern->array, size - 1, Arena_Get(&intern->arena)); + upb_Array_Delete(intern->array, index, 1); } /** diff --git a/php/src/Google/Protobuf/Internal/RepeatedField.php b/php/src/Google/Protobuf/Internal/RepeatedField.php index ea7971f134fe..f6ecd1c84a95 100644 --- a/php/src/Google/Protobuf/Internal/RepeatedField.php +++ b/php/src/Google/Protobuf/Internal/RepeatedField.php @@ -219,13 +219,13 @@ public function offsetSet($offset, $value) public function offsetUnset($offset) { $count = count($this->container); - if (!is_numeric($offset) || $count === 0 || $offset !== $count - 1) { + if (!is_numeric($offset) || $count === 0 || $offset < 0 || $offset >= $count) { trigger_error( "Cannot remove element at the given index", E_USER_ERROR); return; } - array_pop($this->container); + array_splice($this->container, $offset, 1); } /** diff --git a/php/tests/ArrayTest.php b/php/tests/ArrayTest.php index 9e8fcb8beada..2cade79026ea 100644 --- a/php/tests/ArrayTest.php +++ b/php/tests/ArrayTest.php @@ -655,4 +655,31 @@ public function testClone() $arr2 = clone $arr; $this->assertSame($arr[0], $arr2[0]); } + + ######################################################### + # Test offsetUnset + ######################################################### + + public function testOffsetUnset() + { + $arr = new RepeatedField(GPBType::INT32); + $arr[] = 0; + $arr[] = 1; + $arr[] = 2; + + $this->assertSame(3, count($arr)); + $this->assertSame(0, $arr[0]); + $this->assertSame(1, $arr[1]); + $this->assertSame(2, $arr[2]); + + $arr->offsetUnset(1); + $this->assertSame(0, $arr[0]); + $this->assertSame(2, $arr[1]); + + $arr->offsetUnset(0); + $this->assertSame(2, $arr[0]); + + $arr->offsetUnset(0); + $this->assertCount(0, $arr); + } } From 6e00b6b2f57ec0d0c8686afc2b5ed5e78a0ec03d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 10:28:12 -0800 Subject: [PATCH 008/463] Skip bazel build in objectivec/generate_well_known_types.sh. Every where the script is invoked from protoc is already built, and this more follows the model used by similar scripts for other languages. PiperOrigin-RevId: 508694936 --- objectivec/DevTools/full_mac_build.sh | 2 +- objectivec/generate_well_known_types.sh | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index d091a95088c4..84b0aab2ce04 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -204,7 +204,7 @@ else fi # Ensure the WKT sources checked in are current. -time objectivec/generate_well_known_types.sh --check-only $BazelFlags +objectivec/generate_well_known_types.sh --check-only header "Checking on the ObjC Runtime Code" # Some of the kokoro machines don't have python3 yet, so fall back to python if need be. diff --git a/objectivec/generate_well_known_types.sh b/objectivec/generate_well_known_types.sh index 7e23a99d10be..4b77412056d9 100755 --- a/objectivec/generate_well_known_types.sh +++ b/objectivec/generate_well_known_types.sh @@ -9,8 +9,7 @@ readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") readonly ObjCDir="${ScriptDir}" readonly ProtoRootDir="${ObjCDir}/.." -# Invoke with BAZEL=bazelisk to use that instead. -readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}" +cd "${ProtoRootDir}" # Flag for continuous integration to check that everything is current. CHECK_ONLY=0 @@ -19,7 +18,11 @@ if [[ $# -ge 1 && ( "$1" == "--check-only" ) ]] ; then shift fi -cd "${ProtoRootDir}" +readonly PROTOC_PATH="${PROTOC:-${ProtoRootDir}/bazel-bin/protoc}" +if [[ ! -x "${PROTOC_PATH}" ]] ; then + echo "Failed to find executable protoc: ${PROTOC_PATH}" + exit 1 +fi if [[ ! -e src/google/protobuf/stubs/common.h ]]; then cat >&2 << __EOF__ @@ -29,9 +32,6 @@ __EOF__ exit 1 fi -# Make sure the compiler is current. -${BazelBin} build //:protoc $@ - cd src declare -a RUNTIME_PROTO_FILES=( \ google/protobuf/any.proto \ @@ -50,7 +50,7 @@ declare -a OBJC_EXTENSIONS=( .pbobjc.h .pbobjc.m ) # Generate to a temp directory to see if they match. TMP_DIR=$(mktemp -d) trap "rm -rf ${TMP_DIR}" EXIT -${ProtoRootDir}/bazel-bin/protoc --objc_out="${TMP_DIR}" ${RUNTIME_PROTO_FILES[@]} +"${PROTOC_PATH}" --objc_out="${TMP_DIR}" ${RUNTIME_PROTO_FILES[@]} DID_COPY=0 for PROTO_FILE in "${RUNTIME_PROTO_FILES[@]}"; do From 205972b1f75341a65e0cfe0d698e0cfc5260967b Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 10:49:56 -0800 Subject: [PATCH 009/463] Fix remove label action PiperOrigin-RevId: 508701071 --- .github/workflows/test_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index 6ae863c2e6af..f897c65d27c3 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -91,7 +91,7 @@ jobs: steps: - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 with: - labels: safe for tests + labels: ':a: safe for tests' # Note: this pattern of passing the head sha is vulnerable to PWN requests for # pull_request_target events. We carefully limit those workflows to require a From d5ec124bb92b82aff6dbe589a14694924b984575 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 10:51:27 -0800 Subject: [PATCH 010/463] Explicitly ban changes to workflow files from forked PRs Changes to these files *can't* be tested in forked PRs, so we should explicitly block them with an error message to explain why. PiperOrigin-RevId: 508701535 --- .github/README.md | 8 ++++++ .../workflows/forked_pr_workflow_check.yml | 27 +++++++++++++++++++ .github/workflows/test_runner.yml | 4 +-- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/forked_pr_workflow_check.yml diff --git a/.github/README.md b/.github/README.md index b1c780323c79..5c5a6a3a4372 100644 --- a/.github/README.md +++ b/.github/README.md @@ -71,6 +71,14 @@ is working as expected (that is, it will run all the staleness tests). The `regenerate_stale_files.sh` script is the central script responsible for all the re-generation of stale files. +# Forked PRs + +Because we need secret access to run our tests, we use the `pull_request_target` +event for PRs coming from forked repositories. We do checkout the code from the +PR's head, but the workflow files themselves are always fetched from the *base* +branch (that is, the branch we're merging to). Therefore, any changes to these +files won't be tested, so we explicitly ban PRs that touch these files. + # Caches We have a number of different caching strategies to help speed up tests. These diff --git a/.github/workflows/forked_pr_workflow_check.yml b/.github/workflows/forked_pr_workflow_check.yml new file mode 100644 index 000000000000..28be6bbb731e --- /dev/null +++ b/.github/workflows/forked_pr_workflow_check.yml @@ -0,0 +1,27 @@ +name: Forked PR workflow check + +# This workflow prevents modifications to our workflow files in PRs from forked +# repositories. Since tests in these PRs always use the workflows in the +# *target* branch, modifications to these files can't be properly tested. + +on: + # safe presubmit + pull_request: + branches: + - main + - '[0-9]+.x' + # The 21.x branch still uses Kokoro + - '!21.x' + # For testing purposes so we can stage this on the `gha` branch. + - gha + paths: + - '.github/workflows/**' + +jobs: + check: + name: Check PR source + runs-on: ubuntu-latest + steps: + - run: > + ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/protobuf' }} || + (echo "This pull request is from an unsafe fork and isn't allowed to modify workflow files!" && exit 1) diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index f897c65d27c3..3d674013efcd 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -40,8 +40,8 @@ on: branches: - main - '[0-9]+.x' - # The 21.x and 22.x branches still use Kokoro - - '!2[12].x' + # The 21.x branch still use Kokoro + - '!21.x' # For testing purposes so we can stage this on the `gha` branch. - gha types: [labeled, opened, reopened, synchronize] From 1cf2901a1d428fb5101e35b7ac3a44be57226108 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 11:21:07 -0800 Subject: [PATCH 011/463] Use is_closed() in more places in the runtime. PiperOrigin-RevId: 508709514 --- src/google/protobuf/generated_message_reflection.cc | 7 +++---- src/google/protobuf/wire_format.cc | 8 +++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 80a3eb5e914c..41d2d9667e75 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -1339,8 +1339,8 @@ void Reflection::ClearField(Message* message, case FieldDescriptor::CPPTYPE_MESSAGE: if (schema_.HasBitIndex(field) == static_cast(-1)) { - // Proto3 does not have has-bits and we need to set a message field - // to nullptr in order to indicate its un-presence. + // Some fields do not have hasbits and we need to set a message + // field to nullptr in order to indicate its un-presence. if (message->GetArenaForAllocation() == nullptr) { delete *MutableRaw(message, field); } @@ -1554,8 +1554,7 @@ void CheckInOrder(const FieldDescriptor* field, uint32_t* last) { namespace internal { bool CreateUnknownEnumValues(const FieldDescriptor* field) { - bool open_enum = false; - return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || open_enum; + return field->enum_type() == nullptr || !field->enum_type()->is_closed(); } } // namespace internal using internal::CreateUnknownEnumValues; diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index 27aa64fda93a..e27678bf2ad3 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -412,7 +412,7 @@ bool WireFormat::ParseAndMergeMessageSetField(uint32_t field_number, } static bool StrictUtf8Check(const FieldDescriptor* field) { - return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3; + return field->requires_utf8_validation(); } bool WireFormat::ParseAndMergeField( @@ -484,8 +484,7 @@ bool WireFormat::ParseAndMergeField( if (!WireFormatLite::ReadPrimitive( input, &value)) return false; - if (message->GetDescriptor()->file()->syntax() == - FileDescriptor::SYNTAX_PROTO3) { + if (!field->enum_type()->is_closed()) { message_reflection->AddEnumValue(message, field, value); } else { const EnumValueDescriptor* enum_value = @@ -869,8 +868,7 @@ const char* WireFormat::_InternalParseAndMergeField( auto rep_enum = reflection->MutableRepeatedFieldInternal(msg, field); bool open_enum = false; - if (field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || - open_enum) { + if (!field->enum_type()->is_closed() || open_enum) { ptr = internal::PackedEnumParser(rep_enum, ptr, ctx); } else { return ctx->ReadPackedVarint( From 3493da38781eecfd08676a7a29d7294ad480179a Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Fri, 10 Feb 2023 11:35:42 -0800 Subject: [PATCH 012/463] Automated rollback of commit 0b72758dacf68a051e894400950cc2b37b8148eb. PiperOrigin-RevId: 508713070 --- .../src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs index 1fb08db5edd0..78008acd802d 100644 --- a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs +++ b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs @@ -397,11 +397,6 @@ public void DescriptorProtoFileDescriptor() [Test] public void DescriptorImportingExtensionsFromOldCodeGen() { - if (MethodOptions.Descriptor.FullName != "google.protobuf.MethodOptions") - { - Assert.Ignore("Embedded descriptor for OldExtensions expects google.protobuf reflection package."); - } - // The extension collection includes a null extension. There's not a lot we can do about that // in itself, as the old generator didn't provide us the extension information. var extensions = TestProtos.OldGenerator.OldExtensions2Reflection.Descriptor.Extensions; From 9b7f163431a24b7d369e8ae8c77c50c8783ba6b9 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 12:02:42 -0800 Subject: [PATCH 013/463] Fix forked PR check PiperOrigin-RevId: 508720169 --- .github/workflows/forked_pr_workflow_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/forked_pr_workflow_check.yml b/.github/workflows/forked_pr_workflow_check.yml index 28be6bbb731e..f1dd9b55168f 100644 --- a/.github/workflows/forked_pr_workflow_check.yml +++ b/.github/workflows/forked_pr_workflow_check.yml @@ -23,5 +23,5 @@ jobs: runs-on: ubuntu-latest steps: - run: > - ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/protobuf' }} || - (echo "This pull request is from an unsafe fork and isn't allowed to modify workflow files!" && exit 1) + ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/protobuf' }} || + (echo "This pull request is from an unsafe fork (${{ github.event.pull_request.head.repo.full_name }}) and isn't allowed to modify workflow files!" && exit 1) From 31a7fb462d8b0b996b2341ab397263670930d485 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 13:51:06 -0800 Subject: [PATCH 014/463] Internal change PiperOrigin-RevId: 508746384 --- src/google/protobuf/BUILD.bazel | 1 + src/google/protobuf/unknown_field_set_unittest.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index d986479ef796..e3db90cc513e 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -1257,6 +1257,7 @@ cc_test( "//src/google/protobuf/io", "//src/google/protobuf/stubs", "//src/google/protobuf/testing", + "@com_google_absl//absl/functional:bind_front", "@com_google_absl//absl/synchronization", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index e7233947f52a..67b6b3d60f01 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -46,6 +46,7 @@ #include "google/protobuf/testing/googletest.h" #include #include "absl/container/flat_hash_set.h" +#include "absl/functional/bind_front.h" #include "absl/strings/cord.h" #include "absl/synchronization/mutex.h" #include "absl/time/clock.h" From e2aa06eaa833d988a288ff82139c3d7ab1a9cd07 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 14:11:22 -0800 Subject: [PATCH 015/463] [ObjC] Set up a stateless test for the WKTs. Also wire it into the regeneration script. PiperOrigin-RevId: 508751290 --- objectivec/BUILD.bazel | 54 +++++++++++++++++++++++++++++++ objectivec/defs.bzl | 68 +++++++++++++++++++++++++++++++++++++++ regenerate_stale_files.sh | 1 + 3 files changed, 123 insertions(+) create mode 100644 objectivec/defs.bzl diff --git a/objectivec/BUILD.bazel b/objectivec/BUILD.bazel index 3a63685b2f58..fae65c298fb7 100644 --- a/objectivec/BUILD.bazel +++ b/objectivec/BUILD.bazel @@ -1,6 +1,60 @@ load("@rules_cc//cc:defs.bzl", "objc_library") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@upb//cmake:build_defs.bzl", "staleness_test") load("//conformance:defs.bzl", "conformance_test") +load(":defs.bzl", "objc_proto_camel_case_name") + +# The WKTs have to be checked in to support the CocoaPods and Xcode builds. This +# generule and test ensure the source are current. +# +# TODO: Improve the bazel build so it uses these generated headers so it is +# always current, and only the builds that can't easily build protoc and +# generate the files rely on the checked in ones. + +_WELL_KNOWN_TYPES = [ + "any", + "api", + "duration", + "empty", + "field_mask", + "source_context", + "struct", + "timestamp", + "type", + "wrappers", +] + +_OBJC_WKT_NAMES = [objc_proto_camel_case_name(x) for x in _WELL_KNOWN_TYPES] + +_OBJC_EXTS = [ + ".pbobjc.h", + ".pbobjc.m", +] + +genrule( + name = "gen_wkt_sources", + srcs = ["//src/google/protobuf:well_known_type_protos"], + outs = ["wkt/GPB" + wkt + ext for wkt in _OBJC_WKT_NAMES for ext in _OBJC_EXTS], + cmd = " && ".join([ + "$(execpath //:protoc) --objc_out=$(RULEDIR)/wkt --proto_path=src $(SRCS)", + ] + [ + "mv $(RULEDIR)/wkt/google/protobuf/" + wkt + ext + " $(RULEDIR)/wkt/GPB" + wkt + ext + for wkt in _OBJC_WKT_NAMES + for ext in _OBJC_EXTS + ]), + exec_tools = ["//:protoc"], +) + +staleness_test( + name = "well_known_types_staleness_test", + outs = ["GPB" + wkt + ext for wkt in _OBJC_WKT_NAMES for ext in _OBJC_EXTS], + generated_pattern = "wkt/%s", + tags = ["manual"], +) + +################################################################################ +# Objective-C Runtime Library +################################################################################ objc_library( name = "objectivec", diff --git a/objectivec/defs.bzl b/objectivec/defs.bzl new file mode 100644 index 000000000000..5535868bd7eb --- /dev/null +++ b/objectivec/defs.bzl @@ -0,0 +1,68 @@ +"""Starlark helpers for Objective-C protos.""" + +# State constants for objc_proto_camel_case_name. +_last_was_other = 0 +_last_was_lowercase = 1 +_last_was_uppercase = 2 +_last_was_number = 3 + +def objc_proto_camel_case_name(name): + """A Starlark version of the ObjC generator's CamelCase name transform. + + This needs to track + src/google/protobuf/compiler/objectivec/names.cc's UnderscoresToCamelCase() + + NOTE: This code is written to model the C++ in protoc's ObjC generator so it + is easier to confirm that the algorithms between the implementations match. + The customizations for starlark performance are: + - The cascade within the loop is reordered and special cases "_" to + optimize for google3 inputs. + - The "last was" state is handled via integers instead of three booleans. + + The `first_capitalized` argument in the C++ code is left off this code and + it acts as if the value were `True`. + + Args: + name: The proto file name to convert to camel case. The extension should + already be removed. + + Returns: + The converted proto name to camel case. + """ + segments = [] + current = "" + last_was = _last_was_other + for c in name.elems(): + if c.islower(): + # lowercase letter can follow a lowercase or uppercase letter. + if last_was != _last_was_lowercase and last_was != _last_was_uppercase: + segments.append(current) + current = c.upper() + else: + current += c + last_was = _last_was_lowercase + elif c == "_": # more common than rest, special case it. + last_was = _last_was_other + elif c.isdigit(): + if last_was != _last_was_number: + segments.append(current) + current = "" + current += c + last_was = _last_was_number + elif c.isupper(): + if last_was != _last_was_uppercase: + segments.append(current) + current = c + else: + current += c.lower() + last_was = _last_was_uppercase + else: + last_was = _last_was_other + segments.append(current) + result = "" + for x in segments: + if x in ("Url", "Http", "Https"): + result += x.upper() + else: + result += x + return result diff --git a/regenerate_stale_files.sh b/regenerate_stale_files.sh index 1c38104135cd..58c640b38b56 100755 --- a/regenerate_stale_files.sh +++ b/regenerate_stale_files.sh @@ -15,6 +15,7 @@ readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS}" # Run and fix all staleness tests. ${BazelBin} test src:cmake_lists_staleness_test "$@" || ./bazel-bin/src/cmake_lists_staleness_test --fix ${BazelBin} test src/google/protobuf:well_known_types_staleness_test "$@" || ./bazel-bin/src/google/protobuf/well_known_types_staleness_test --fix +${BazelBin} test objectivec:well_known_types_staleness_test "$@" || ./bazel-bin/objectivec/well_known_types_staleness_test --fix # Generate C# code. # This doesn't currently have Bazel staleness tests, but there's an existing From f239739a71c8fc5a58a1b19d81728e2e74595a95 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 14:33:29 -0800 Subject: [PATCH 016/463] Have the CocoaPods CI update the WKTs before testing. PiperOrigin-RevId: 508756680 --- .github/workflows/test_objectivec.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index f5720cd24521..f572d6d19df0 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -73,11 +73,14 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Pod lib lint - run: | - pod lib lint --verbose \ - --configuration=${{ matrix.CONFIGURATION }} \ - --platforms=${{ matrix.PLATFORM }} \ - Protobuf.podspec + uses: ./.github/actions/bash + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: | + pod lib lint --verbose \ + --configuration=${{ matrix.CONFIGURATION }} \ + --platforms=${{ matrix.PLATFORM }} \ + Protobuf.podspec bazel: strategy: From 79a4f9476ffb27a5bace8b923cc9bded4ac27f1c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 Feb 2023 14:59:49 -0800 Subject: [PATCH 017/463] [ObjC] Move the Xcode CI off full_mac_build.sh Use the bash GHA to ensure WKTs are current, and then just directly invoke Xcode. This means the script full_mac_build.sh can go back to just being a simpler helper for folks doing local builds and we can remove all the extra support that was needed for CI. PiperOrigin-RevId: 508762464 --- .github/BUILD.bazel | 6 ++++- .github/workflows/test_objectivec.yml | 37 +++++++++++++++------------ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/BUILD.bazel b/.github/BUILD.bazel index 98a95d1bb849..12daec4a3725 100644 --- a/.github/BUILD.bazel +++ b/.github/BUILD.bazel @@ -1,5 +1,9 @@ # This information is extracted from the MacOS runner specs located at: -# https://github.com/actions/runner-images/blob/win19/20230129.2/images/macos/macos-12-Readme.md +# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md +# +# When updating, also ensure the "xcode_destination" entries in +# `.github/workflows/test_objectivec.yml` are supported for the given versions +# of Xcode. xcode_version( name = "version14_2_14C18", version = "14.2.14C18", diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index f572d6d19df0..465e02259231 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -13,20 +13,19 @@ jobs: strategy: fail-fast: false # Don't cancel all jobs if one fails. matrix: + platform: ["macOS", "iOS"] + xc_config: ["Debug", "Release"] + # The "destination" entries need to match what is available for the selected Xcode. + # See `.github/BUILD.bazel` for the Xcode info. include: - - name: macOS - config: osx - flags: --skip-xcode-ios --skip-xcode-tvos --skip-objc-conformance - # The iOS simulator takes a while to start up, so Debug & Release are run in - # parallel to get the testing done faster. - - name: iOS Debug - config: ios_debug - flags: --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-release - - name: iOS Release - config: ios_release - flags: --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-debug + - platform: "macOS" + destination: "platform=macOS" + xc_project: "ProtocolBuffers_OSX.xcodeproj" + - platform: "iOS" + destination: "platform=iOS Simulator,name=iPhone 13,OS=latest" + xc_project: "ProtocolBuffers_iOS.xcodeproj" - name: Xcode ${{ matrix.name}} + name: Xcode ${{ matrix.platform}} ${{ matrix.xc_config }} runs-on: macos-12 env: DEVELOPER_DIR: /Applications/Xcode_14.1.app/Contents/Developer @@ -39,18 +38,24 @@ jobs: - name: Setup ccache uses: ./.github/actions/ccache with: - cache-prefix: objectivec_macos_${{ matrix.config }} + cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }} support-modules: true - name: Run tests - uses: ./.github/actions/bazel + uses: ./.github/actions/bash env: CC: ${{ github.workspace }}/ci/clang_wrapper CXX: ${{ github.workspace }}/ci/clang_wrapper++ with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: objectivec_macos/${{ matrix.config }} - bash: objectivec/DevTools/full_mac_build.sh ${{ matrix.flags }} + command: | + xcodebuild \ + -project "objectivec/${{ matrix.xc_project }}" \ + -scheme ProtocolBuffers \ + -configuration ${{ matrix.xc_config }} \ + -destination "${{ matrix.destination }}" \ + test \ + | xcpretty - name: Report ccache stats shell: bash From 9c2211be08af10dd5964cf8b65364afaef001fdf Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 18:24:16 -0800 Subject: [PATCH 018/463] Switch to Ninja generator for windows cmake builds. These will still use MSVC as the compiler, but will no longer generate Visual Studio projects for the builds. Visual Studio is particularly bad at parallelizing builds, and is hostile to ccache. This change also tweaks the ccache setup to prevent unbounded the growth observed in our github caches. Windows builds have had debug symbols stripped to reduce ccache size by a factor of 2x, and ccache maximum have been tweaked so that we persist fewer older builds. Before this change, each CMake build took 12 minutes on every run (plus some constant overhead from staleness/gcloud), even with caching or on large multi-core runners. No amount of caching or parallelization made any noticeable difference above noise. With this change, we see the following improvements: - 12 minutes to build from scratch on normal runners (unchanged) - 4 minutes on 32-core runners from scratch - 1 minute with optimal caches available on normal runners. - 30 seconds on 32-core runners with optimal caches PiperOrigin-RevId: 508799909 --- .github/actions/ccache/action.yml | 36 ++-- .../internal/ccache-setup-windows/action.yml | 28 +-- .github/workflows/test_cpp.yml | 166 ++++++------------ CMakeLists.txt | 16 +- 4 files changed, 90 insertions(+), 156 deletions(-) diff --git a/.github/actions/ccache/action.yml b/.github/actions/ccache/action.yml index 2a100ccb7ff5..2d705503649d 100644 --- a/.github/actions/ccache/action.yml +++ b/.github/actions/ccache/action.yml @@ -12,6 +12,18 @@ inputs: runs: using: 'composite' steps: + - name: Configure ccache environment variables + shell: bash + run: | + echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV + echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV + echo "CCACHE_DIRECT=true" >> $GITHUB_ENV + echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV + - name: Setup ccache on Windows if: ${{ runner.os == 'Windows' }} uses: ./.github/actions/internal/ccache-setup-windows @@ -23,7 +35,10 @@ runs: - name: Setup fixed path ccache caching uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 with: - path: .ccache + path: | + .ccache/** + !.ccache/lock + !.ccache/tmp # Always push to a cache key unique to this commit. key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} # Select a cache to restore from with the follow order of preference: @@ -35,18 +50,6 @@ runs: ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }} ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} - - name: Configure ccache environment variables - shell: bash - run: | - echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=600M" >> $GITHUB_ENV - echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV - echo "CCACHE_DIRECT=true" >> $GITHUB_ENV - echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache $CCACHE_CMAKE_FLAGS" >> $GITHUB_ENV - - name: Enable module support if: ${{ inputs.support-modules }} shell: bash @@ -55,11 +58,6 @@ runs: echo "CCACHE_DEPEND=true" >> $GITHUB_ENV - name: Zero out ccache - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os != 'Linux' }} shell: bash run: ccache -z - - - name: Zero out ccache - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: ${{ github.workspace }}\ccache.exe -z diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml index 5479cd328034..a42e7bf8a8ed 100644 --- a/.github/actions/internal/ccache-setup-windows/action.yml +++ b/.github/actions/internal/ccache-setup-windows/action.yml @@ -10,6 +10,16 @@ inputs: runs: using: 'composite' steps: + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 + with: + arch: x64 + vsversion: '2019' + + - name: Install ccache + shell: bash + run: choco install ccache --version=4.7.4 + - name: Configure ccache environment variables shell: pwsh run: | @@ -18,19 +28,9 @@ runs: echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - name: Download ccache + - name: Configure Windows-specific ccache environment variables shell: bash + # Windows caches are about 2x larger than other platforms. run: | - curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip - unzip ccache.zip - cp ccache-${{ inputs.ccache-version }}-windows-x86_64/ccache.exe ccache.exe - cp ccache.exe cl.exe - rm ccache.zip - - - name: Configure msbuild flags - shell: bash - run: echo "CCACHE_MSBUILD_FLAGS=/p:CLToolExe=cl.exe /p:CLToolPath=${{ github.workspace}}" >> $GITHUB_ENV - - - name: Configure cmake flags - shell: bash - run: echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 5d51bcae3c5d..f846d80a8dff 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -179,49 +179,34 @@ jobs: bazel: test ${{ matrix.bazel }} bazel-cache: cpp_${{ matrix.os }} - macos-cmake: - name: MacOS CMake - runs-on: macos-12 - steps: - - name: Checkout pending changes - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - submodules: recursive - ref: ${{ inputs.safe-checkout }} - - - name: Setup ccache - uses: ./.github/actions/ccache - with: - cache-prefix: macos-cmake - - - name: Configure CMake - uses: ./.github/actions/bash - with: - credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} - - name: Build - run: cmake --build . --parallel 8 - - name: Test - run: ctest --verbose --parallel 20 -C Debug - - - name: Report ccache stats - shell: bash - run: ccache -s -v - - windows-cmake: + non-linux-cmake: strategy: fail-fast: false # Don't cancel all jobs if one fails. matrix: include: - - name: Visual Studio + - name: MacOS CMake + os: macos-12 + flags: -DCMAKE_CXX_STANDARD=14 + - name: Windows CMake + os: windows-2019 flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_EXAMPLES=ON - - name: Shared - flags: -Dprotobuf_BUILD_SHARED_LIBS=ON - - name: Windows CMake ${{ matrix.name}} - runs-on: windows-2019 + - name: Windows CMake Shared + os: windows-2019 + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_BUILD_SHARED_LIBS=ON + - name: Windows CMake Install + os: windows-2019 + install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON + -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} steps: - name: Checkout pending changes uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 @@ -229,102 +214,49 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - name: Setup ccache uses: ./.github/actions/ccache with: - cache-prefix: windows-cmake-${{ matrix.name }} + cache-prefix: ${{ matrix.name }} - - name: Configure CMake + # Install phase. + - name: Configure CMake for install + if: matrix.install-flags uses: ./.github/actions/bash with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: | - cmake . -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF \ - ${{ matrix.flags }} - - - name: Build for Windows 15 2017 - run: >- - msbuild.exe protobuf.sln /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true - /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 - ${{ env.CCACHE_MSBUILD_FLAGS }} - - - name: Run Tests - run: ctest --verbose --parallel 20 -C Debug - - - name: Report ccache stats - run: ${{ github.workspace }}\ccache.exe -s -v - - windows-cmake-install: - name: Windows CMake Install - runs-on: windows-2019 - steps: - - name: Checkout pending changes - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - submodules: recursive - ref: ${{ inputs.safe-checkout }} - - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - - name: Setup ccache - uses: ./.github/actions/ccache - with: - cache-prefix: windows-cmake + command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }} + - name: Build for install + if: matrix.install-flags + shell: bash + run: VERBOSE=1 cmake --build . --parallel 20 + - name: Install + if: matrix.install-flags + shell: bash + run: cmake --build . --target install + - name: Report and clear ccache stats + if: matrix.install-flags + shell: bash + run: ccache -s -v && ccache -z + - name: Clear CMake cache + if: matrix.install-flags + shell: bash + run: cmake --build . --target clean && rm CMakeCache.txt - - name: Configure CMake for Install + - name: Configure CMake uses: ./.github/actions/bash with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: | - mkdir build - pushd build - cmake .. -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF - popd - - - name: Build and Install Protobuf for Windows 15 2017 - run: | - pushd build - msbuild.exe INSTALL.vcxproj /p:Platform=x64 /p:VisualStudioVersion=15.0 /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true ${{ env.CCACHE_MSBUILD_FLAGS }} - popd + command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }} - - name: Clear CMake cache + - name: Build shell: bash - run: rm -rf build/* + run: VERBOSE=1 cmake --build . --parallel 20 - - name: Configure CMake + - name: Test shell: bash - run: | - cmake . -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \ - -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF - - - name: Build for Windows 15 2017 - run: >- - msbuild.exe protobuf.sln /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true - /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 - ${{ env.CCACHE_MSBUILD_FLAGS }} - - - name: Run Tests run: ctest --verbose --parallel 20 -C Debug - name: Report ccache stats - run: ${{ github.workspace }}\ccache.exe -s -v + shell: bash + run: ccache -s -v diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c5513ca29dc..20786bd3ff4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,13 +292,17 @@ if (MSVC) string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") if (protobuf_ALLOW_CCACHE) - # In order to support ccache, we replace the /Zi option with /Z7. This + # In order to support ccache, we need to remove the /Zi option because it + # puts debug symbols into separate pdb files (which in incompatible with + # ccache). This can be replaced with /Z7 to preserve debug symbols, which # embeds debug symbols into the object files instead of creating a separate - # pdb file, which isn't currently supported by ccache. - string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") - string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") - string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") - string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") + # pdb file, which isn't currently supported by ccache. However, this bloats + # the ccache size by about a factor of 2x, making it very expensive in CI. + # Instead, we strip debug symbols to reduce this overhead. + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") endif() # Suppress linker warnings about files with no symbols defined. From 364328858e2694f4f85613cf168a96053eb434d6 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Sat, 11 Feb 2023 17:21:20 -0800 Subject: [PATCH 019/463] Tweak GHA names to make failures clearer in dashboard PiperOrigin-RevId: 508944353 --- .github/workflows/staleness_check.yml | 2 +- .github/workflows/staleness_refresh.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 7ae33d22ded0..022d64eb3905 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -14,7 +14,7 @@ jobs: branch: [main, 22.x] os: [{ name: Linux, value: ubuntu-latest}] - name: ${{ matrix.os.name }} ${{ matrix.branch}} + name: Test staleness ${{ matrix.os.name }} ${{ matrix.branch}} runs-on: ${{ matrix.os.value }} steps: - name: Checkout ${{ matrix.branch }} diff --git a/.github/workflows/staleness_refresh.yml b/.github/workflows/staleness_refresh.yml index 5264b14b80f7..8f083fdc6b9b 100644 --- a/.github/workflows/staleness_refresh.yml +++ b/.github/workflows/staleness_refresh.yml @@ -11,12 +11,13 @@ on: permissions: {} jobs: - cmake: + run: permissions: contents: write # for git push if: github.repository == 'protocolbuffers/protobuf' runs-on: ubuntu-latest + name: Refresh stale files strategy: fail-fast: false # Don't cancel all jobs if one fails. From 50861195140be676fb46ddacba635a78640f4e10 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Sun, 12 Feb 2023 19:35:55 -0800 Subject: [PATCH 020/463] Move GHA documentation into workflows directory. Placing it in the .github directory unexpectedly replaced our top-level documentation on the main page. PiperOrigin-RevId: 509104326 --- .github/{ => workflows}/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/README.md (100%) diff --git a/.github/README.md b/.github/workflows/README.md similarity index 100% rename from .github/README.md rename to .github/workflows/README.md From 4e76952c76c186b80343d91ff5bf662404025aa1 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Sun, 12 Feb 2023 21:49:12 -0800 Subject: [PATCH 021/463] Stop using 'push' event to determine privileged post-submit runs. Both 'schedule' and 'workflow_dispatch' are valid alternatives today. What we really want here is anything *except* pull request events. PiperOrigin-RevId: 509123777 --- .github/actions/bazel-docker/action.yml | 2 +- .github/actions/bazel/action.yml | 6 +++--- .github/workflows/test_runner.yml | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/bazel-docker/action.yml b/.github/actions/bazel-docker/action.yml index 800d8eca24b9..af1104e23df8 100644 --- a/.github/actions/bazel-docker/action.yml +++ b/.github/actions/bazel-docker/action.yml @@ -71,5 +71,5 @@ runs: - name: Save Bazel repository cache # Only allow repository cache updates during post-submits. - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml index a848cfc394aa..141527db0267 100644 --- a/.github/actions/bazel/action.yml +++ b/.github/actions/bazel/action.yml @@ -63,14 +63,14 @@ runs: run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV - name: Cache Bazelisk - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 with: path: ${{ env.BAZELISK_PATH }} key: bazel-${{ runner.os }}-${{ inputs.version }} - name: Restore Bazelisk - if: ${{ github.event_name != 'push' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 with: path: ${{ env.BAZELISK_PATH }} @@ -107,5 +107,5 @@ runs: - name: Save Bazel repository cache # Only allow repository cache updates during post-submits. - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target'}} uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index 3d674013efcd..9318af007752 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -57,7 +57,8 @@ jobs: # repository, it's safe and we can use `pull_request`. Otherwise, we should # use `pull_request_target`. if: | - (github.event_name == 'push' && + (github.event_name != 'pull_request' && + github.event_name != 'pull_request_target' && github.event.repository.full_name == 'protocolbuffers/protobuf') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'protocolbuffers/protobuf') || From e45e66b5f421b65f2aa032c50d0506dbc129e27c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 Feb 2023 10:07:13 -0800 Subject: [PATCH 022/463] [ObjC] CI dbg & opt macOS bazel builds. PiperOrigin-RevId: 509260795 --- .github/workflows/test_objectivec.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 465e02259231..4093b180b724 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -91,11 +91,15 @@ jobs: strategy: fail-fast: false # Don't cancel all jobs if one fails. matrix: + config: + - { name: Optimized, flags: --config=opt } + - { name: Debug, flags: --config=dbg } + # TODO: Could add iOS to atleast build the objc_library targets for that. + platform: ["macOS"] include: - # TODO: Could add iOS to atleast build the objc_library targets for that. - - name: macOS - bazel: //objectivec/... - name: Bazel ${{ matrix.name }} + - platform: "macOS" + bazel_targets: //objectivec/... + name: Bazel ${{ matrix.platform }} ${{ matrix.config.name }} runs-on: macos-12 steps: - name: Checkout pending changes @@ -106,5 +110,5 @@ jobs: uses: ./.github/actions/bazel with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel: test ${{ matrix.bazel }} - bazel-cache: objc_${{ matrix.name }} + bazel: test ${{ matrix.config.flags }} ${{ matrix.bazel_targets }} + bazel-cache: objc_${{ matrix.platform }}_${{ matrix.config.name }} From a2c5ea614cd5330de4b56a9937377f65309a31f8 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 Feb 2023 10:09:00 -0800 Subject: [PATCH 023/463] [ObjC] make the bazel build always use generated WKTs. This is done in basically the same way the C++ does this. It was tested locally be putting `#error` directives in the pre-generated sources and running the bazel build (it worked). PiperOrigin-RevId: 509261352 --- objectivec/BUILD.bazel | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/objectivec/BUILD.bazel b/objectivec/BUILD.bazel index fae65c298fb7..0965de5b276f 100644 --- a/objectivec/BUILD.bazel +++ b/objectivec/BUILD.bazel @@ -7,9 +7,11 @@ load(":defs.bzl", "objc_proto_camel_case_name") # The WKTs have to be checked in to support the CocoaPods and Xcode builds. This # generule and test ensure the source are current. # -# TODO: Improve the bazel build so it uses these generated headers so it is -# always current, and only the builds that can't easily build protoc and -# generate the files rely on the checked in ones. +# Within the ":objectivec" target, the outputs of the genrule are then used to +# ensure they are always "current". This implementation is basically the same +# has how the WKTs are handled in src/google/protobuf/BUILD.bazel for the C++ +# version. They share the potential downsides around layer checks and that +# someone could #include the header with the `wkt/` prefix on the name. _WELL_KNOWN_TYPES = [ "any", @@ -59,16 +61,6 @@ staleness_test( objc_library( name = "objectivec", hdrs = [ - "GPBAny.pbobjc.h", - "GPBApi.pbobjc.h", - "GPBDuration.pbobjc.h", - "GPBEmpty.pbobjc.h", - "GPBFieldMask.pbobjc.h", - "GPBSourceContext.pbobjc.h", - "GPBStruct.pbobjc.h", - "GPBTimestamp.pbobjc.h", - "GPBType.pbobjc.h", - "GPBWrappers.pbobjc.h", "GPBArray.h", "GPBBootstrap.h", "GPBCodedInputStream.h", @@ -109,39 +101,30 @@ objc_library( "GPBUnknownFieldSet_PackagePrivate.h", "GPBUnknownField_PackagePrivate.h", "GPBUtilities_PackagePrivate.h", - ], + ] + ["wkt/GPB" + wkt + ".pbobjc.h" for wkt in _OBJC_WKT_NAMES], copts = [ "-Wno-vla", ], includes = [ ".", + "wkt", ], non_arc_srcs = [ - "GPBAny.pbobjc.m", - "GPBApi.pbobjc.m", "GPBArray.m", "GPBCodedInputStream.m", "GPBCodedOutputStream.m", "GPBDescriptor.m", "GPBDictionary.m", - "GPBDuration.pbobjc.m", - "GPBEmpty.pbobjc.m", "GPBExtensionInternals.m", "GPBExtensionRegistry.m", - "GPBFieldMask.pbobjc.m", "GPBMessage.m", "GPBRootObject.m", - "GPBSourceContext.pbobjc.m", - "GPBStruct.pbobjc.m", - "GPBTimestamp.pbobjc.m", - "GPBType.pbobjc.m", "GPBUnknownField.m", "GPBUnknownFieldSet.m", "GPBUtilities.m", "GPBWellKnownTypes.m", "GPBWireFormat.m", - "GPBWrappers.pbobjc.m", - ], + ] + ["wkt/GPB" + wkt + ".pbobjc.m" for wkt in _OBJC_WKT_NAMES], target_compatible_with = select({ "@platforms//os:macos": [], "@platforms//os:ios": [], @@ -159,8 +142,8 @@ objc_library( conformance_test( name = "conformance_test", failure_list = "//conformance:failure_list_objc.txt", - testee = "//conformance:conformance_objc", target_compatible_with = ["@platforms//os:macos"], + testee = "//conformance:conformance_objc", ) # ------------------------------------------------------------------- From 28c0a48b5ec73188a3ddbde784776980de339c73 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 Feb 2023 10:29:57 -0800 Subject: [PATCH 024/463] [ObjC] full_mac_builds.sh has been update - The support for CI has been removed (no longer needed) - Update some of the testing targets - Removed some lesser used options (to simplify) PiperOrigin-RevId: 509268013 --- objectivec/DevTools/full_mac_build.sh | 33 ++++++--------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index 84b0aab2ce04..16981f9ee896 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -2,8 +2,6 @@ # # Helper to do build so you don't have to remember all the steps/args. -echo "::group::Run full mac build" - set -eu # Some base locations. @@ -12,7 +10,7 @@ readonly ProtoRootDir="${ScriptDir}/../.." readonly BazelFlags="${BAZEL_FLAGS:---announce_rc --macos_minimum_os=10.9}" # Invoke with BAZEL=bazelisk to use that instead. -readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}" +readonly BazelBin="${BAZEL:-bazel}" printUsage() { NAME=$(basename "${0}") @@ -29,9 +27,6 @@ OPTIONS: Show this message -c, --clean Issue a clean before the normal build. - -r, --regenerate-descriptors - Run generate_descriptor_proto.sh to regenerate all the checked in - proto sources. --full-build By default only protoc is built within protobuf, this option will enable a full build/test of the entire protobuf project. @@ -76,7 +71,6 @@ else fi DO_CLEAN=no -REGEN_DESCRIPTORS=no FULL_BUILD=no DO_XCODE_IOS_TESTS=yes DO_XCODE_OSX_TESTS=yes @@ -94,9 +88,6 @@ while [[ $# != 0 ]]; do -c | --clean ) DO_CLEAN=yes ;; - -r | --regenerate-descriptors ) - REGEN_DESCRIPTORS=yes - ;; --full-build ) FULL_BUILD=yes ;; @@ -190,17 +181,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then fi fi -if [[ "${REGEN_DESCRIPTORS}" == "yes" ]] ; then - header "Regenerating the descriptor sources." - ./generate_descriptor_proto.sh -fi - if [[ "${FULL_BUILD}" == "yes" ]] ; then header "Build/Test: everything" - time ${BazelBin} test //:protoc //:protobuf //src/... $BazelFlags + ${BazelBin} test //:protoc //:protobuf //src/... $BazelFlags else header "Building: protoc" - time ${BazelBin} build //:protoc $BazelFlags + ${BazelBin} build //:protoc $BazelFlags fi # Ensure the WKT sources checked in are current. @@ -233,8 +219,7 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then if [[ "${XCODE_QUIET}" == "yes" ]] ; then XCODEBUILD_TEST_BASE_IOS+=( -quiet ) fi - # Don't need to worry about form factors or retina/non retina; - # just pick a mix of OS Versions and 32/64 bit. + # Don't need to worry about form factors or retina/non retina. # NOTE: Different Xcode have different simulated hardware/os support. case "${XCODE_VERSION}" in [6-9].* | 1[0-2].* ) @@ -242,9 +227,8 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then exit 11 ;; 13.* | 14.*) - # Dropped 32bit as Apple doesn't seem support the simulators either. XCODEBUILD_TEST_BASE_IOS+=( - -destination "platform=iOS Simulator,name=iPhone 8,OS=latest" # 64bit + -destination "platform=iOS Simulator,name=iPhone 13,OS=latest" ) ;; * ) @@ -271,8 +255,7 @@ if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then "${XCODEBUILD}" -project objectivec/ProtocolBuffers_OSX.xcodeproj -scheme ProtocolBuffers - # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported. - -destination "platform=OS X,arch=x86_64" # 64bit + -destination "platform=macOS" ) if [[ "${XCODE_QUIET}" == "yes" ]] ; then XCODEBUILD_TEST_BASE_OSX+=( -quiet ) @@ -331,10 +314,8 @@ fi if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then header "Running ObjC Conformance Tests" - time ${BazelBin} test //objectivec:conformance_test $BazelFlags + ${BazelBin} test //objectivec:conformance_test $BazelFlags fi echo "" echo "$(basename "${0}"): Success!" - -echo "::endgroup::" From 08bedfdbe4390aa62aafb6f1d784348b8fce2ef8 Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Mon, 13 Feb 2023 11:41:24 -0800 Subject: [PATCH 025/463] Disable std::string specialization PiperOrigin-RevId: 509289178 --- src/google/protobuf/arena.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index e9e7a15d4e4d..23e6999c4c0c 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -681,11 +681,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { friend struct internal::ArenaTestPeer; }; -template <> -inline void* Arena::AllocateInternal() { - return impl_.AllocateFromStringBlock(); -} - } // namespace protobuf } // namespace google From 853a55ebbbbc4424fb1d56a31fe07e1ec961c9cd Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Mon, 13 Feb 2023 13:30:20 -0800 Subject: [PATCH 026/463] Cleanup FastTV32S1() and FastTV64S1() functions. - replaces the existing FastV32S1() and FastV64S1() implementations with the FastTV32S1() and FastTV64S1() functions, removing the latter names, and moves the internal template helpers from the header into the .cc file. - defines PROTOBUF_TC_PARAM_NO_DATA_DECL and PROTOBUF_TC_PARAM_NO_DATA_PASS macros to declare functions not using `data`, and to use in call sites calling these functions. END_PUBLIC - adds MUSTTAIL to ToParseLoop() and Error(). The current code was inconsistent on this, and if it doesn't help, it doesn't hurt either and minimizes any possible polition from the call sites. PiperOrigin-RevId: 509317845 --- .../protobuf/generated_message_tctable_decl.h | 23 +- .../protobuf/generated_message_tctable_impl.h | 233 +----------- .../generated_message_tctable_lite.cc | 349 +++++++++++++----- .../generated_message_tctable_lite_test.cc | 15 +- src/google/protobuf/port_def.inc | 13 + 5 files changed, 318 insertions(+), 315 deletions(-) diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index dd4845ba886d..4ec6973d2332 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -54,6 +54,7 @@ namespace internal { // Additional information about this field: struct TcFieldData { constexpr TcFieldData() : data(0) {} + explicit constexpr TcFieldData(uint64_t data) : data(data) {} // Fast table entry constructor: constexpr TcFieldData(uint16_t coded_tag, uint8_t hasbit_idx, uint8_t aux_idx, @@ -63,6 +64,24 @@ struct TcFieldData { uint64_t{hasbit_idx} << 16 | // uint64_t{coded_tag}) {} + // Constructor to create an explicit 'uninitialized' instance. + // This constructor can be used to pass an uninitialized `data` value to a + // table driven parser function that does not use `data`. The purpose of this + // is that it allows the compiler to reallocate and re-purpose the register + // that is currently holding its value for other data. This reduces register + // allocations inside the highly optimized varint parsing functions. + // + // Applications not using `data` use the `PROTOBUF_TC_PARAM_NO_DATA_DECL` + // macro to declare the standard input arguments with no name for the `data` + // argument. Callers then use the `PROTOBUF_TC_PARAM_NO_DATA_PASS` macro. + // + // Example: + // if (ptr == nullptr) { + // PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + // } + struct DefaultInit {}; + TcFieldData(DefaultInit) {} // NOLINT(google-explicit-constructor) + // Fields used in fast table parsing: // // Bit: @@ -122,7 +141,9 @@ struct TcFieldData { uint32_t tag() const { return static_cast(data); } uint32_t entry_offset() const { return static_cast(data >> 32); } - uint64_t data; + union { + uint64_t data; + }; }; struct TcParseTableBase; diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 4f2cc2a51619..84ee0e12841a 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -292,7 +292,7 @@ class PROTOBUF_EXPORT TcParser final { // the function is used as a way to get a UnknownFieldOps vtable, returned // via the `const char*` return type. See `GetUnknownFieldOps()` - static bool MustFallbackToGeneric(PROTOBUF_TC_PARAM_DECL) { + static bool MustFallbackToGeneric(PROTOBUF_TC_PARAM_NO_DATA_DECL) { return ptr == nullptr; } @@ -359,22 +359,16 @@ class PROTOBUF_EXPORT TcParser final { static const char* FastZ64P1(PROTOBUF_TC_PARAM_DECL); static const char* FastZ64P2(PROTOBUF_TC_PARAM_DECL); - // Manually unrolled and specialized Varint parsing. - template - static const char* FastTV32S1(PROTOBUF_TC_PARAM_DECL); - template - static const char* FastTV64S1(PROTOBUF_TC_PARAM_DECL); - template static constexpr TailCallParseFunc SingularVarintNoZag1() { if (sizeof(FieldType) == 1) { return &FastV8S1; } if (sizeof(FieldType) == 4) { - return &FastTV32S1; + return &FastV32S1; } if (sizeof(FieldType) == 8) { - return &FastTV64S1; + return &FastV64S1; } static_assert(sizeof(FieldType) == 1 || sizeof(FieldType) == 4 || sizeof(FieldType) == 8, @@ -515,12 +509,16 @@ class PROTOBUF_EXPORT TcParser final { // NOTE: Currently, this function only calls the table-level fallback // function, so it should only be called as the fallback from fast table // parsing. - static const char* MiniParse(PROTOBUF_TC_PARAM_DECL); + static const char* MiniParse(PROTOBUF_TC_PARAM_NO_DATA_DECL); static const char* FastEndG1(PROTOBUF_TC_PARAM_DECL); static const char* FastEndG2(PROTOBUF_TC_PARAM_DECL); private: + // Optimized small tag varint parser for int32/int64 + template + static const char* FastVarintS1(PROTOBUF_TC_PARAM_DECL); + friend class GeneratedTcTableLiteTest; static void* MaybeGetSplitBase(MessageLite* msg, const bool is_split, const TcParseTableBase* table); @@ -555,10 +553,10 @@ class PROTOBUF_EXPORT TcParser final { } } - static const char* TagDispatch(PROTOBUF_TC_PARAM_DECL); - static const char* ToTagDispatch(PROTOBUF_TC_PARAM_DECL); - static const char* ToParseLoop(PROTOBUF_TC_PARAM_DECL); - static const char* Error(PROTOBUF_TC_PARAM_DECL); + static const char* TagDispatch(PROTOBUF_TC_PARAM_NO_DATA_DECL); + static const char* ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_DECL); + static const char* ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_DECL); + static const char* Error(PROTOBUF_TC_PARAM_NO_DATA_DECL); static const char* FastUnknownEnumFallback(PROTOBUF_TC_PARAM_DECL); @@ -712,205 +710,14 @@ class PROTOBUF_EXPORT TcParser final { static const char* MpFallback(PROTOBUF_TC_PARAM_DECL); }; -// Shift "byte" left by n * 7 bits, filling vacated bits with ones. -template -inline PROTOBUF_ALWAYS_INLINE int64_t shift_left_fill_with_ones(uint64_t byte, - uint64_t ones) { - return static_cast((byte << (n * 7)) | (ones >> (64 - (n * 7)))); -} - -// Shift "byte" left by n * 7 bits, filling vacated bits with ones, and -// put the new value in res. Return whether the result was negative. -template -inline PROTOBUF_ALWAYS_INLINE bool shift_left_fill_with_ones_was_negative( - uint64_t byte, uint64_t ones, int64_t& res) { -#if defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__) - // For the first two rounds (ptr[1] and ptr[2]), micro benchmarks show a - // substantial improvement from capturing the sign from the condition code - // register on x86-64. - bool sign_bit; - asm("shldq %3, %2, %1" - : "=@ccs"(sign_bit), "+r"(byte) - : "r"(ones), "i"(n * 7)); - res = static_cast(byte); - return sign_bit; -#else - // Generic fallback: - res = shift_left_fill_with_ones(byte, ones); - return res < 0; -#endif -} - -template -inline PROTOBUF_ALWAYS_INLINE std::pair -ParseFallbackPair(const char* p, int64_t res1) { - constexpr bool kIs64BitVarint = std::is_same::value; - constexpr bool kIs32BitVarint = std::is_same::value; - static_assert(kIs64BitVarint || kIs32BitVarint, - "Only 32 or 64 bit varints are supported"); - auto ptr = reinterpret_cast(p); - - // The algorithm relies on sign extension for each byte to set all high bits - // when the varint continues. It also relies on asserting all of the lower - // bits for each successive byte read. This allows the result to be aggregated - // using a bitwise AND. For example: - // - // 8 1 64 57 ... 24 17 16 9 8 1 - // ptr[0] = 1aaa aaaa ; res1 = 1111 1111 ... 1111 1111 1111 1111 1aaa aaaa - // ptr[1] = 1bbb bbbb ; res2 = 1111 1111 ... 1111 1111 11bb bbbb b111 1111 - // ptr[2] = 1ccc cccc ; res3 = 0000 0000 ... 000c cccc cc11 1111 1111 1111 - // --------------------------------------------- - // res1 & res2 & res3 = 0000 0000 ... 000c cccc ccbb bbbb baaa aaaa - // - // On x86-64, a shld from a single register filled with enough 1s in the high - // bits can accomplish all this in one instruction. It so happens that res1 - // has 57 high bits of ones, which is enough for the largest shift done. - // - // Just as importantly, by keeping results in res1, res2, and res3, we take - // advantage of the superscalar abilities of the CPU. - ABSL_DCHECK_EQ(res1 >> 7, -1); - uint64_t ones = res1; // save the high 1 bits from res1 (input to SHLD) - int64_t res2, res3; // accumulated result chunks - - if (!shift_left_fill_with_ones_was_negative<1>(ptr[1], ones, res2)) - goto done2; - if (!shift_left_fill_with_ones_was_negative<2>(ptr[2], ones, res3)) - goto done3; - - // For the remainder of the chunks, check the sign of the AND result. - res2 &= shift_left_fill_with_ones<3>(ptr[3], ones); - if (res2 >= 0) goto done4; - res1 &= shift_left_fill_with_ones<4>(ptr[4], ones); - if (res1 >= 0) goto done5; - if (kIs64BitVarint) { - res2 &= shift_left_fill_with_ones<5>(ptr[5], ones); - if (res2 >= 0) goto done6; - res3 &= shift_left_fill_with_ones<6>(ptr[6], ones); - if (res3 >= 0) goto done7; - res1 &= shift_left_fill_with_ones<7>(ptr[7], ones); - if (res1 >= 0) goto done8; - res3 &= shift_left_fill_with_ones<8>(ptr[8], ones); - if (res3 >= 0) goto done9; - } else if (kIs32BitVarint) { - if (PROTOBUF_PREDICT_TRUE(!(ptr[5] & 0x80))) goto done6; - if (PROTOBUF_PREDICT_TRUE(!(ptr[6] & 0x80))) goto done7; - if (PROTOBUF_PREDICT_TRUE(!(ptr[7] & 0x80))) goto done8; - if (PROTOBUF_PREDICT_TRUE(!(ptr[8] & 0x80))) goto done9; - } - - // For valid 64bit varints, the 10th byte/ptr[9] should be exactly 1. In this - // case, the continuation bit of ptr[8] already set the top bit of res3 - // correctly, so all we have to do is check that the expected case is true. - if (PROTOBUF_PREDICT_TRUE(kIs64BitVarint && ptr[9] == 1)) goto done10; - - if (PROTOBUF_PREDICT_FALSE(ptr[9] & 0x80)) { - // If the continue bit is set, it is an unterminated varint. - return {nullptr, 0}; - } - - // A zero value of the first bit of the 10th byte represents an - // over-serialized varint. This case should not happen, but if does (say, due - // to a nonconforming serializer), deassert the continuation bit that came - // from ptr[8]. - if (kIs64BitVarint && (ptr[9] & 1) == 0) { -#if defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__) - // Use a small instruction since this is an uncommon code path. - asm("btcq $63,%0" : "+r"(res3)); -#else - res3 ^= static_cast(1) << 63; -#endif - } - goto done10; - -done2: - return {p + 2, res1 & res2}; -done3: - return {p + 3, res1 & res2 & res3}; -done4: - return {p + 4, res1 & res2 & res3}; -done5: - return {p + 5, res1 & res2 & res3}; -done6: - return {p + 6, res1 & res2 & res3}; -done7: - return {p + 7, res1 & res2 & res3}; -done8: - return {p + 8, res1 & res2 & res3}; -done9: - return {p + 9, res1 & res2 & res3}; -done10: - return {p + 10, res1 & res2 & res3}; -} - -template -PROTOBUF_NOINLINE const char* TcParser::FastTV64S1(PROTOBUF_TC_PARAM_DECL) { - using TagType = uint8_t; - // super-early success test... - if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { - ptr += sizeof(TagType); // Consume tag - hasbits |= (uint64_t{1} << data.hasbit_idx()); - uint8_t value = data.data >> 8; - RefAt(msg, data.offset()) = value; - ptr += 1; - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); - } - if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); - } - ptr += sizeof(TagType); // Consume tag - hasbits |= (uint64_t{1} << data.hasbit_idx()); - - auto tmp = - ParseFallbackPair(ptr, static_cast(data.data >> 8)); - ptr = tmp.first; - if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { - data.data = 0; // Indicate to the compiler that we don't need this anymore. - return Error(PROTOBUF_TC_PARAM_PASS); - } - - RefAt(msg, data.offset()) = static_cast(tmp.second); - data.data = 0; // Indicate to the compiler that we don't need this anymore. - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); -} - -template -PROTOBUF_NOINLINE const char* TcParser::FastTV32S1(PROTOBUF_TC_PARAM_DECL) { - using TagType = uint8_t; - // super-early success test... - if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { - ptr += sizeof(TagType); // Consume tag - hasbits |= (uint64_t{1} << data.hasbit_idx()); - uint8_t value = data.data >> 8; - RefAt(msg, data.offset()) = value; - ptr += 1; - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); - } - if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); - } - ptr += sizeof(TagType); // Consume tag - hasbits |= (uint64_t{1} << data.hasbit_idx()); - - auto tmp = - ParseFallbackPair(ptr, static_cast(data.data >> 8)); - ptr = tmp.first; - if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { - return Error(PROTOBUF_TC_PARAM_PASS); - } - - RefAt(msg, data.offset()) = static_cast(tmp.second); - data.data = 0; // Indicate to the compiler that we don't need this anymore. - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); -} - // Dispatch to the designated parse function inline PROTOBUF_ALWAYS_INLINE const char* TcParser::TagDispatch( - PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_TC_PARAM_NO_DATA_DECL) { const auto coded_tag = UnalignedLoad(ptr); const size_t idx = coded_tag & table->fast_idx_mask; PROTOBUF_ASSUME((idx & 7) == 0); auto* fast_entry = table->fast_entry(idx >> 3); - data = fast_entry->bits; + TcFieldData data = fast_entry->bits; data.data ^= coded_tag; PROTOBUF_MUSTTAIL return fast_entry->target()(PROTOBUF_TC_PARAM_PASS); } @@ -921,25 +728,23 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::TagDispatch( // mode. Luckily the structure of the algorithm is such that it's always // possible to just return and use the enclosing parse loop as a trampoline. inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToTagDispatch( - PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_TC_PARAM_NO_DATA_DECL) { constexpr bool always_return = !PROTOBUF_TAILCALL; if (always_return || !ctx->DataAvailable(ptr)) { - PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } - PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToParseLoop( - PROTOBUF_TC_PARAM_DECL) { - (void)data; + PROTOBUF_TC_PARAM_NO_DATA_DECL) { (void)ctx; SyncHasbits(msg, hasbits, table); return ptr; } inline PROTOBUF_ALWAYS_INLINE const char* TcParser::Error( - PROTOBUF_TC_PARAM_DECL) { - (void)data; + PROTOBUF_TC_PARAM_NO_DATA_DECL) { (void)ctx; (void)ptr; SyncHasbits(msg, hasbits, table); diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index acc1915efa8e..1455a9e286ba 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -98,7 +98,7 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop( // TODO(b/64614992): remove this asm asm("" : "+r"(table)); #endif - ptr = TagDispatch(msg, ptr, ctx, {}, table - 1, 0); + ptr = TagDispatch(msg, ptr, ctx, TcFieldData::DefaultInit(), table - 1, 0); if (ptr == nullptr) break; if (ctx->LastTag() != 1) break; // Ended on terminating tag } @@ -279,7 +279,7 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::MiniParse( ptr = ReadTagInlined(ptr, &tag); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { if (export_called_function) *test_out = {Error}; - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } auto* entry = FindFieldEntry(table, tag >> 3); @@ -351,8 +351,9 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::MiniParse( PROTOBUF_MUSTTAIL return parse_fn(PROTOBUF_TC_PARAM_PASS); } -PROTOBUF_NOINLINE const char* TcParser::MiniParse(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +PROTOBUF_NOINLINE const char* TcParser::MiniParse( + PROTOBUF_TC_PARAM_NO_DATA_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE TcParser::TestMiniParseResult TcParser::TestMiniParse( PROTOBUF_TC_PARAM_DECL) { @@ -369,11 +370,11 @@ const char* TcParser::MpFallback(PROTOBUF_TC_PARAM_DECL) { template const char* TcParser::FastEndGroupImpl(PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } ctx->SetLastTag(data.decoded_tag()); ptr += sizeof(TagType); - PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastEndG1(PROTOBUF_TC_PARAM_DECL) { @@ -403,7 +404,7 @@ template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); @@ -478,7 +479,7 @@ template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const auto expected_tag = UnalignedLoad(ptr); const auto aux = *table->field_aux(data.aux_idx()); @@ -502,14 +503,14 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( } } if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { - PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { - PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } while (UnalignedLoad(ptr) == expected_tag); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { @@ -560,13 +561,13 @@ template PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularFixed( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } ptr += sizeof(TagType); // Consume tag hasbits |= (uint64_t{1} << data.hasbit_idx()); RefAt(msg, data.offset()) = UnalignedLoad(ptr); ptr += sizeof(LayoutType); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastF32S1(PROTOBUF_TC_PARAM_DECL) { @@ -598,7 +599,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedFixed( if (data.coded_tag() == 0) { return PackedFixed(PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } auto& field = RefAt>(msg, data.offset()); @@ -607,7 +608,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedFixed( field.Add(UnalignedLoad(ptr + sizeof(TagType))); ptr += sizeof(TagType) + sizeof(LayoutType); } while (ctx->DataAvailable(ptr) && UnalignedLoad(ptr) == tag); - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastF32R1(PROTOBUF_TC_PARAM_DECL) { @@ -643,7 +644,7 @@ const char* TcParser::PackedFixed(PROTOBUF_TC_PARAM_DECL) { if (data.coded_tag() == 0) { return RepeatedFixed(PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } ptr += sizeof(TagType); @@ -680,6 +681,136 @@ PROTOBUF_NOINLINE const char* TcParser::FastF64P2(PROTOBUF_TC_PARAM_DECL) { namespace { +// Shift "byte" left by n * 7 bits, filling vacated bits with ones. +template +inline PROTOBUF_ALWAYS_INLINE int64_t shift_left_fill_with_ones(uint64_t byte, + uint64_t ones) { + return static_cast((byte << (n * 7)) | (ones >> (64 - (n * 7)))); +} + +// Shift "byte" left by n * 7 bits, filling vacated bits with ones, and +// put the new value in res. Return whether the result was negative. +template +inline PROTOBUF_ALWAYS_INLINE bool shift_left_fill_with_ones_was_negative( + uint64_t byte, uint64_t ones, int64_t& res) { +#if defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__) + // For the first two rounds (up to 2 varint bytes), micro benchmarks show a + // substantial improvement from capturing the sign from the condition code + // register on x86-64. + bool sign_bit; + asm("shldq %3, %2, %1" + : "=@ccs"(sign_bit), "+r"(byte) + : "r"(ones), "i"(n * 7)); + res = static_cast(byte); + return sign_bit; +#else + // Generic fallback: + res = shift_left_fill_with_ones(byte, ones); + return res < 0; +#endif +} + +template +inline PROTOBUF_ALWAYS_INLINE std::pair +ParseFallbackPair(const char* p, int64_t res1) { + constexpr bool kIs64BitVarint = std::is_same::value; + constexpr bool kIs32BitVarint = std::is_same::value; + static_assert(kIs64BitVarint || kIs32BitVarint, + "Only 32 or 64 bit varints are supported"); + auto ptr = reinterpret_cast(p); + + // The algorithm relies on sign extension for each byte to set all high bits + // when the varint continues. It also relies on asserting all of the lower + // bits for each successive byte read. This allows the result to be aggregated + // using a bitwise AND. For example: + // + // 8 1 64 57 ... 24 17 16 9 8 1 + // ptr[0] = 1aaa aaaa ; res1 = 1111 1111 ... 1111 1111 1111 1111 1aaa aaaa + // ptr[1] = 1bbb bbbb ; res2 = 1111 1111 ... 1111 1111 11bb bbbb b111 1111 + // ptr[2] = 0ccc cccc ; res3 = 0000 0000 ... 000c cccc cc11 1111 1111 1111 + // --------------------------------------------- + // res1 & res2 & res3 = 0000 0000 ... 000c cccc ccbb bbbb baaa aaaa + // + // On x86-64, a shld from a single register filled with enough 1s in the high + // bits can accomplish all this in one instruction. It so happens that res1 + // has 57 high bits of ones, which is enough for the largest shift done. + // + // Just as importantly, by keeping results in res1, res2, and res3, we take + // advantage of the superscalar abilities of the CPU. + ABSL_DCHECK_EQ(res1 >> 7, -1); + uint64_t ones = res1; // save the high 1 bits from res1 (input to SHLD) + int64_t res2, res3; // accumulated result chunks + + if (!shift_left_fill_with_ones_was_negative<1>(ptr[1], ones, res2)) + goto done2; + if (!shift_left_fill_with_ones_was_negative<2>(ptr[2], ones, res3)) + goto done3; + + // For the remainder of the chunks, check the sign of the AND result. + res2 &= shift_left_fill_with_ones<3>(ptr[3], ones); + if (res2 >= 0) goto done4; + res1 &= shift_left_fill_with_ones<4>(ptr[4], ones); + if (res1 >= 0) goto done5; + if (kIs64BitVarint) { + res2 &= shift_left_fill_with_ones<5>(ptr[5], ones); + if (res2 >= 0) goto done6; + res3 &= shift_left_fill_with_ones<6>(ptr[6], ones); + if (res3 >= 0) goto done7; + res1 &= shift_left_fill_with_ones<7>(ptr[7], ones); + if (res1 >= 0) goto done8; + res3 &= shift_left_fill_with_ones<8>(ptr[8], ones); + if (res3 >= 0) goto done9; + } else if (kIs32BitVarint) { + if (PROTOBUF_PREDICT_TRUE(!(ptr[5] & 0x80))) goto done6; + if (PROTOBUF_PREDICT_TRUE(!(ptr[6] & 0x80))) goto done7; + if (PROTOBUF_PREDICT_TRUE(!(ptr[7] & 0x80))) goto done8; + if (PROTOBUF_PREDICT_TRUE(!(ptr[8] & 0x80))) goto done9; + } + + // For valid 64bit varints, the 10th byte/ptr[9] should be exactly 1. In this + // case, the continuation bit of ptr[8] already set the top bit of res3 + // correctly, so all we have to do is check that the expected case is true. + if (PROTOBUF_PREDICT_TRUE(kIs64BitVarint && ptr[9] == 1)) goto done10; + + if (PROTOBUF_PREDICT_FALSE(ptr[9] & 0x80)) { + // If the continue bit is set, it is an unterminated varint. + return {nullptr, 0}; + } + + // A zero value of the first bit of the 10th byte represents an + // over-serialized varint. This case should not happen, but if does (say, due + // to a nonconforming serializer), deassert the continuation bit that came + // from ptr[8]. + if (kIs64BitVarint && (ptr[9] & 1) == 0) { +#if defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__) + // Use a small instruction since this is an uncommon code path. + asm("btcq $63,%0" : "+r"(res3)); +#else + res3 ^= static_cast(1) << 63; +#endif + } + goto done10; + +done2: + return {p + 2, res1 & res2}; +done3: + return {p + 3, res1 & res2 & res3}; +done4: + return {p + 4, res1 & res2 & res3}; +done5: + return {p + 5, res1 & res2 & res3}; +done6: + return {p + 6, res1 & res2 & res3}; +done7: + return {p + 7, res1 & res2 & res3}; +done8: + return {p + 8, res1 & res2 & res3}; +done9: + return {p + 9, res1 & res2 & res3}; +done10: + return {p + 10, res1 & res2 & res3}; +} + template inline PROTOBUF_ALWAYS_INLINE const char* ParseVarint(const char* p, Type* value) { @@ -792,7 +923,7 @@ template PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularVarint( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } ptr += sizeof(TagType); // Consume tag hasbits |= (uint64_t{1} << data.hasbit_idx()); @@ -807,7 +938,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularVarint( RefAt(msg, data.offset()) = ZigZagDecodeHelper(static_cast(*ptr++)); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } template @@ -842,11 +973,41 @@ PROTOBUF_NOINLINE const char* TcParser::SingularVarBigint( hasbits = spill.hasbits; if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } RefAt(msg, data.offset()) = ZigZagDecodeHelper(tmp); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); +} + +template +PROTOBUF_ALWAYS_INLINE const char* TcParser::FastVarintS1( + PROTOBUF_TC_PARAM_DECL) { + using TagType = uint8_t; + // super-early success test... + if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { + ptr += sizeof(TagType); // Consume tag + hasbits |= (uint64_t{1} << data.hasbit_idx()); + uint8_t value = data.data >> 8; + RefAt(msg, data.offset()) = value; + ptr += 1; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + ptr += sizeof(TagType); // Consume tag + hasbits |= (uint64_t{1} << data.hasbit_idx()); + + auto tmp = + ParseFallbackPair(ptr, static_cast(data.data >> 8)); + ptr = tmp.first; + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + + RefAt(msg, data.offset()) = tmp.second; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV8S1(PROTOBUF_TC_PARAM_DECL) { @@ -870,13 +1031,13 @@ PROTOBUF_NOINLINE const char* TcParser::FastV8S1(PROTOBUF_TC_PARAM_DECL) { ptr += sizeof(TagType) + 1; // Consume the tag and the value. hasbits |= (uint64_t{1} << data.hasbit_idx()); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } // If it didn't match above either the tag is wrong, or the value is encoded // non-canonically. // Jump to MiniParse as wrong tag is the most probable reason. - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV8S2(PROTOBUF_TC_PARAM_DECL) { @@ -884,16 +1045,14 @@ PROTOBUF_NOINLINE const char* TcParser::FastV8S2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV32S1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularVarint( - PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return FastVarintS1(PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV32S2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return SingularVarint( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV64S1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularVarint( - PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return FastVarintS1(PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV64S2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return SingularVarint( @@ -926,7 +1085,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedVarint( if (data.coded_tag() == 0) { return PackedVarint(PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } auto& field = RefAt>(msg, data.offset()); @@ -936,14 +1095,14 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedVarint( FieldType tmp; ptr = ParseVarint(ptr, &tmp); if (ptr == nullptr) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } field.Add(ZigZagDecodeHelper(tmp)); if (!ctx->DataAvailable(ptr)) { break; } } while (UnalignedLoad(ptr) == expected_tag); - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastV8R1(PROTOBUF_TC_PARAM_DECL) { @@ -996,7 +1155,7 @@ const char* TcParser::PackedVarint(PROTOBUF_TC_PARAM_DECL) { if (data.coded_tag() == 0) { return RepeatedVarint(PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } ptr += sizeof(TagType); @@ -1071,7 +1230,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback( uint32_t tag; ptr = ReadTag(ptr, &tag); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } data.data = tag; PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); @@ -1081,14 +1240,14 @@ template PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularEnum( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* ptr2 = ptr; // Save for unknown enum case ptr += sizeof(TagType); // Consume tag uint64_t tmp; ptr = ParseVarint(ptr, &tmp); if (ptr == nullptr) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const TcParseTableBase::FieldAux aux = *table->field_aux(data.aux_idx()); if (PROTOBUF_PREDICT_FALSE( @@ -1098,7 +1257,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularEnum( } hasbits |= (uint64_t{1} << data.hasbit_idx()); RefAt(msg, data.offset()) = tmp; - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastErS1(PROTOBUF_TC_PARAM_DECL) { @@ -1126,7 +1285,7 @@ const char* TcParser::RepeatedEnum(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return PackedEnum( PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } auto& field = RefAt>(msg, data.offset()); @@ -1138,7 +1297,7 @@ const char* TcParser::RepeatedEnum(PROTOBUF_TC_PARAM_DECL) { uint64_t tmp; ptr = ParseVarint(ptr, &tmp); if (ptr == nullptr) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (PROTOBUF_PREDICT_FALSE( !EnumIsValidAux(static_cast(tmp), xform_val, aux))) { @@ -1152,7 +1311,7 @@ const char* TcParser::RepeatedEnum(PROTOBUF_TC_PARAM_DECL) { break; } } while (UnalignedLoad(ptr) == expected_tag); - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const TcParser::UnknownFieldOps& TcParser::GetUnknownFieldOps( @@ -1180,7 +1339,7 @@ const char* TcParser::PackedEnum(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return RepeatedEnum( PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } const auto saved_tag = UnalignedLoad(ptr); @@ -1237,18 +1396,18 @@ template PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularEnumSmallRange( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } uint8_t v = ptr[sizeof(TagType)]; if (PROTOBUF_PREDICT_FALSE(min > v || v > data.aux_idx())) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } RefAt(msg, data.offset()) = v; ptr += sizeof(TagType) + 1; hasbits |= (uint64_t{1} << data.hasbit_idx()); - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastEr0S1(PROTOBUF_TC_PARAM_DECL) { @@ -1279,7 +1438,7 @@ const char* TcParser::RepeatedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return PackedEnumSmallRange( PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } auto& field = RefAt>(msg, data.offset()); @@ -1288,14 +1447,14 @@ const char* TcParser::RepeatedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) { do { uint8_t v = ptr[sizeof(TagType)]; if (PROTOBUF_PREDICT_FALSE(min > v || v > max)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } field.Add(static_cast(v)); ptr += sizeof(TagType) + 1; if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) break; } while (UnalignedLoad(ptr) == expected_tag); - PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastEr0R1(PROTOBUF_TC_PARAM_DECL) { @@ -1324,7 +1483,7 @@ const char* TcParser::PackedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return RepeatedEnumSmallRange( PROTOBUF_TC_PARAM_PASS); } else { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } @@ -1414,7 +1573,7 @@ template PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularString( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); @@ -1427,20 +1586,24 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularString( } else { ptr = ReadStringNoArena(msg, ptr, ctx, data.aux_idx(), table, field); } - if (ptr == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } switch (utf8) { case kNoUtf8: #ifdef NDEBUG case kUtf8ValidateOnly: #endif - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); default: if (PROTOBUF_PREDICT_TRUE(IsValidUTF8(field))) { - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } ReportFastUtf8Error(FastDecodeTag(saved_tag), table); - return utf8 == kUtf8 ? Error(PROTOBUF_TC_PARAM_PASS) - : ToParseLoop(PROTOBUF_TC_PARAM_PASS); + if (utf8 == kUtf8) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } } @@ -1474,49 +1637,49 @@ PROTOBUF_NOINLINE const char* TcParser::FastUS2(PROTOBUF_TC_PARAM_DECL) { // Inlined string variants: const char* TcParser::FastBiS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastBiS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastSiS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastSiS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastUiS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastUiS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } // Corded string variants: const char* TcParser::FastBcS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastBcS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastScS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastScS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastUcS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const char* TcParser::FastUcS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } template PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedString( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { - PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS); } const auto expected_tag = UnalignedLoad(ptr); auto& field = RefAt(msg, data.offset()); @@ -1549,7 +1712,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedString( ptr = ParseRepeatedStringOnce(ptr, arena, serial_arena, ctx, field); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr || !validate_last_string())) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (!ctx->DataAvailable(ptr)) break; } while (UnalignedLoad(ptr) == expected_tag); @@ -1559,12 +1722,12 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedString( std::string* str = field.Add(); ptr = InlineGreedyStringParser(str, ptr, ctx); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr || !validate_last_string())) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (!ctx->DataAvailable(ptr)) break; } while (UnalignedLoad(ptr) == expected_tag); } - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastBR1(PROTOBUF_TC_PARAM_DECL) { @@ -1739,7 +1902,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) { RefAt(base, entry.offset) = UnalignedLoad(ptr); ptr += sizeof(uint32_t); } - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed( @@ -1788,7 +1951,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed( } while (next_tag == decoded_tag); } - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::MpPackedFixed(PROTOBUF_TC_PARAM_DECL) { @@ -1817,9 +1980,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedFixed(PROTOBUF_TC_PARAM_DECL) { } if (ptr == nullptr) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } template @@ -1844,7 +2007,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) { const char* ptr2 = ptr; // save for unknown enum case uint64_t tmp; ptr = ParseVarint(ptr, &tmp); - if (ptr == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } // Transform and/or validate the value uint16_t rep = type_card & field_layout::kRepMask; @@ -1881,7 +2046,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) { RefAt(base, entry.offset) = static_cast(tmp); } - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( @@ -1911,11 +2076,15 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( do { uint64_t tmp; ptr = ParseVarint(ptr2, &tmp); - if (ptr == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } field.Add(is_zigzag ? WireFormatLite::ZigZagDecode64(tmp) : tmp); if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); - if (ptr2 == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr2 == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } } while (next_tag == decoded_tag); } else if (rep == field_layout::kRep32Bits) { auto& field = RefAt>(msg, entry.offset); @@ -1924,7 +2093,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( do { uint64_t tmp; ptr = ParseVarint(ptr2, &tmp); - if (ptr == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } if (is_validated_enum) { if (!EnumIsValidAux(tmp, xform_val, *table->field_aux(&entry))) { ptr = ptr2; @@ -1936,7 +2107,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( field.Add(tmp); if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); - if (ptr2 == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr2 == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } } while (next_tag == decoded_tag); } else { ABSL_DCHECK_EQ(rep, static_cast(field_layout::kRep8Bits)); @@ -1946,15 +2119,19 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( do { uint64_t tmp; ptr = ParseVarint(ptr2, &tmp); - if (ptr == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } field.Add(static_cast(tmp)); if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); - if (ptr2 == nullptr) return Error(PROTOBUF_TC_PARAM_PASS); + if (ptr2 == nullptr) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } } while (next_tag == decoded_tag); } - PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) { @@ -2005,7 +2182,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) { ptr, [field](uint64_t value) { field->Add(value); }); } - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } bool TcParser::MpVerifyUtf8(absl::string_view wire_bytes, @@ -2083,9 +2260,9 @@ PROTOBUF_NOINLINE const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) { } if (ptr == nullptr || !is_valid) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } PROTOBUF_ALWAYS_INLINE const char* TcParser::ParseRepeatedStringOnce( @@ -2132,7 +2309,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString( if (PROTOBUF_PREDICT_FALSE(ptr == nullptr || !MpVerifyUtf8(field[field.size() - 1], table, entry, xform_val))) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); @@ -2145,7 +2322,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString( if (PROTOBUF_PREDICT_FALSE( ptr == nullptr || !MpVerifyUtf8(*str, table, entry, xform_val))) { - return Error(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); @@ -2162,7 +2339,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString( #endif } - return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); } template diff --git a/src/google/protobuf/generated_message_tctable_lite_test.cc b/src/google/protobuf/generated_message_tctable_lite_test.cc index 0d923aeb4a27..8aeacf57a28f 100644 --- a/src/google/protobuf/generated_message_tctable_lite_test.cc +++ b/src/google/protobuf/generated_message_tctable_lite_test.cc @@ -125,8 +125,7 @@ TEST(FastVarints, NameHere) { // clang-format on uint8_t serialize_buffer[64]; - // TODO(b/27721823): cleanup test cases for 'former' TV* functions. - for (int size : {8, 32, 64, -8, -32, -64}) { + for (int size : {8, 32, 64}) { SCOPED_TRACE(size); auto next_i = [](uint64_t i) { // if i + 1 is a power of two, return that. @@ -193,21 +192,12 @@ TEST(FastVarints, NameHere) { case 8: fn = &TcParser::FastV8S1; break; - case -8: - fn = &TcParser::FastV8S1; - break; case 32: fn = &TcParser::FastV32S1; break; - case -32: - fn = &TcParser::FastTV32S1; - break; case 64: fn = &TcParser::FastV64S1; break; - case -64: - fn = &TcParser::FastTV64S1; - break; } fallback_ptr_received = absl::nullopt; fallback_hasbits_received = absl::nullopt; @@ -216,7 +206,6 @@ TEST(FastVarints, NameHere) { Xor2SerializedBytes(parse_table.fast_entries[0].bits, ptr), &parse_table.header, /*hasbits=*/0); switch (size) { - case -8: case 8: { if (end_ptr == nullptr) { // If end_ptr is nullptr, that means the FastParser gave up and @@ -241,7 +230,6 @@ TEST(FastVarints, NameHere) { EXPECT_EQ(actual_field, static_cast(i)) // << " hex: " << absl::StrCat(absl::Hex(actual_field)); }; break; - case -32: case 32: { ASSERT_TRUE(end_ptr); ASSERT_EQ(end_ptr - ptr, serialized.size()); @@ -250,7 +238,6 @@ TEST(FastVarints, NameHere) { EXPECT_EQ(actual_field, static_cast(i)) // << " hex: " << absl::StrCat(absl::Hex(actual_field)); }; break; - case -64: case 64: { ASSERT_EQ(end_ptr - ptr, serialized.size()); diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index c38f7a218d8b..d87b6b992906 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -856,6 +856,19 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // PROTOBUF_TC_PARAM_PASS passes values to match PROTOBUF_TC_PARAM_DECL. #define PROTOBUF_TC_PARAM_PASS msg, ptr, ctx, data, table, hasbits +// PROTOBUF_TC_PARAM_NO_DATA_DECL and PROTOBUF_TC_PARAM_NO_DATA_PASS provide the +// exact same ABI as above, except that they don't name or pass the `data` +// argument. Specific functions such as `Error() and `ToTagDispatch()` don't +// use the `data` argument. By not passing `data` down the call stack, we free +// up the register holding that value, which may matter in highly optimized +// functions such as varint parsing. +#define PROTOBUF_TC_PARAM_NO_DATA_DECL \ + ::google::protobuf::MessageLite *msg, const char *ptr, \ + ::google::protobuf::internal::ParseContext *ctx, ::google::protobuf::internal::TcFieldData, \ + const ::google::protobuf::internal::TcParseTableBase *table, uint64_t hasbits +#define PROTOBUF_TC_PARAM_NO_DATA_PASS \ + msg, ptr, ctx, ::google::protobuf::internal::TcFieldData::DefaultInit(), table, hasbits + #ifdef PROTOBUF_UNUSED #error PROTOBUF_UNUSED was previously defined #endif From 8cfd02f590cf9455c22a5812a41dce0663e2e48a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 Feb 2023 13:37:38 -0800 Subject: [PATCH 027/463] Add tracing for limit push/pop in sanitizer mode to make sure we never read an uninitialized one, and we never fail to pop one. Fix a couple of cases where invalid input lead to undefined behavior. PiperOrigin-RevId: 509319815 --- src/google/protobuf/message_unittest.inc | 25 +++++++ src/google/protobuf/parse_context.cc | 15 ++--- src/google/protobuf/parse_context.h | 86 ++++++++++++++++++------ 3 files changed, 96 insertions(+), 30 deletions(-) diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 204da69b4808..c41d8b3d0e78 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -1166,6 +1166,31 @@ TEST(MESSAGE_TEST_NAME, PreservesFloatingPointNegative0) { std::signbit(out_message.optional_double())); } +TEST(MESSAGE_TEST_NAME, + RegressionTestForParseMessageReadingUninitializedLimit) { + UNITTEST::TestAllTypes in_message; + in_message.mutable_optional_nested_message(); + std::string serialized = in_message.SerializeAsString(); + // We expect this to have 3 bytes: two for the tag, and one for the zero size. + // Break the size by making it overlong. + ASSERT_EQ(serialized.size(), 3); + serialized.back() = '\200'; + serialized += std::string(10, '\200'); + EXPECT_FALSE(in_message.ParseFromString(serialized)); +} + +TEST(MESSAGE_TEST_NAME, + RegressionTestForParseMessageWithSizeBeyondInputFailsToPopLimit) { + UNITTEST::TestAllTypes in_message; + in_message.mutable_optional_nested_message(); + std::string serialized = in_message.SerializeAsString(); + // We expect this to have 3 bytes: two for the tag, and one for the zero size. + // Make the size a valid varint, but it overflows in the input. + ASSERT_EQ(serialized.size(), 3); + serialized.back() = 10; + EXPECT_FALSE(in_message.ParseFromString(serialized)); +} + const uint8_t* SkipTag(const uint8_t* buf) { while (*buf & 0x80) ++buf; ++buf; diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index 11d21819fc24..b96e7ac74a6d 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -319,26 +319,19 @@ const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { } const char* ParseContext::ReadSizeAndPushLimitAndDepth(const char* ptr, - int* old_limit) { - int size = ReadSize(&ptr); - if (PROTOBUF_PREDICT_FALSE(!ptr) || depth_ <= 0) { - *old_limit = 0; // Make sure this isn't uninitialized even on error return - return nullptr; - } - *old_limit = PushLimit(ptr, size); - --depth_; - return ptr; + LimitToken* old_limit) { + return ReadSizeAndPushLimitAndDepthInlined(ptr, old_limit); } const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) { - int old; + LimitToken old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); if (ptr == nullptr) return ptr; auto old_depth = depth_; ptr = msg->_InternalParse(ptr, this); if (ptr != nullptr) ABSL_DCHECK_EQ(old_depth, depth_); depth_++; - if (!PopLimit(old)) return nullptr; + if (!PopLimit(std::move(old))) return nullptr; return ptr; } diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index e5b13a027ea5..0ddaa2604bf0 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -36,11 +36,13 @@ #include #include +#include "absl/base/config.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/cord.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "google/protobuf/arena.h" #include "google/protobuf/arenastring.h" #include "google/protobuf/endian.h" @@ -131,8 +133,53 @@ class PROTOBUF_EXPORT EpsCopyInputStream { if (count > 0) StreamBackUp(count); } +#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || defined(ABSL_HAVE_MEMORY_SANITIZER) + // In sanitizer mode we use an optional to guarantee that: + // - We do not read an uninitialized token. + // - Every non-empty token is moved from and consumed. + class LimitToken { + public: + LimitToken() = default; + explicit LimitToken(int token) : token_(token) {} + LimitToken(LimitToken&& other) { *this = std::move(other); } + LimitToken& operator=(LimitToken&& other) { + token_ = std::exchange(other.token_, absl::nullopt); + return *this; + } + + ~LimitToken() { ABSL_CHECK(!token_.has_value()); } + + LimitToken(const LimitToken&) = delete; + LimitToken& operator=(const LimitToken&) = delete; + + int token() && { + ABSL_CHECK(token_.has_value()); + return *std::exchange(token_, absl::nullopt); + } + + private: + absl::optional token_; + }; +#else + class LimitToken { + public: + LimitToken() = default; + explicit LimitToken(int token) : token_(token) {} + LimitToken(LimitToken&&) = default; + LimitToken& operator=(LimitToken&&) = default; + + LimitToken(const LimitToken&) = delete; + LimitToken& operator=(const LimitToken&) = delete; + + int token() const { return token_; } + + private: + int token_; + }; +#endif + // If return value is negative it's an error - PROTOBUF_NODISCARD int PushLimit(const char* ptr, int limit) { + PROTOBUF_NODISCARD LimitToken PushLimit(const char* ptr, int limit) { ABSL_DCHECK(limit >= 0 && limit <= INT_MAX - kSlopBytes); // This add is safe due to the invariant above, because // ptr - buffer_end_ <= kSlopBytes. @@ -140,12 +187,14 @@ class PROTOBUF_EXPORT EpsCopyInputStream { limit_end_ = buffer_end_ + (std::min)(0, limit); auto old_limit = limit_; limit_ = limit; - return old_limit - limit; + return LimitToken(old_limit - limit); } - PROTOBUF_NODISCARD bool PopLimit(int delta) { + PROTOBUF_NODISCARD bool PopLimit(LimitToken delta) { + // We must update the limit first before the early return. Otherwise, we can + // end up with an invalid limit and it can lead to integer overflows. + limit_ = limit_ + std::move(delta).token(); if (PROTOBUF_PREDICT_FALSE(!EndedAtLimit())) return false; - limit_ = limit_ + delta; // TODO(gerbens) We could remove this line and hoist the code to // DoneFallback. Study the perf/bin-size effects. limit_end_ = buffer_end_ + (std::min)(0, limit_); @@ -470,13 +519,14 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { template PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage( MessageLite* msg, const char* ptr, const Table* table) { - int old; + LimitToken old; ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old); + if (ptr == nullptr) return ptr; auto old_depth = depth_; - ptr = ptr ? TcParser::ParseLoop(msg, ptr, this, table) : nullptr; + ptr = TcParser::ParseLoop(msg, ptr, this, table); if (ptr != nullptr) ABSL_DCHECK_EQ(old_depth, depth_); depth_++; - if (!PopLimit(old)) return nullptr; + if (!PopLimit(std::move(old))) return nullptr; return ptr; } @@ -518,20 +568,20 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { private: // Out-of-line routine to save space in ParseContext::ParseMessage - // int old; + // LimitToken old; // ptr = ReadSizeAndPushLimitAndDepth(ptr, &old) // is equivalent to: // int size = ReadSize(&ptr); // if (!ptr) return nullptr; - // int old = PushLimit(ptr, size); + // LimitToken old = PushLimit(ptr, size); // if (--depth_ < 0) return nullptr; - PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth(const char* ptr, - int* old_limit); + PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth( + const char* ptr, LimitToken* old_limit); // As above, but fully inlined for the cases where we care about performance // more than size. eg TcParser. PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* - ReadSizeAndPushLimitAndDepthInlined(const char* ptr, int* old_limit); + ReadSizeAndPushLimitAndDepthInlined(const char* ptr, LimitToken* old_limit); // The context keeps an internal stack to keep track of the recursive // part of the parse state. @@ -881,27 +931,25 @@ template ::value, bool>::type> PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg, const char* ptr) { - int old; + LimitToken old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); if (ptr == nullptr) return ptr; auto old_depth = depth_; ptr = msg->_InternalParse(ptr, this); if (ptr != nullptr) ABSL_DCHECK_EQ(old_depth, depth_); depth_++; - if (!PopLimit(old)) return nullptr; + if (!PopLimit(std::move(old))) return nullptr; return ptr; } inline const char* ParseContext::ReadSizeAndPushLimitAndDepthInlined( - const char* ptr, int* old_limit) { + const char* ptr, LimitToken* old_limit) { int size = ReadSize(&ptr); - if (PROTOBUF_PREDICT_FALSE(!ptr)) { - // Make sure this isn't uninitialized even on error return - *old_limit = 0; + if (PROTOBUF_PREDICT_FALSE(!ptr) || depth_ <= 0) { return nullptr; } *old_limit = PushLimit(ptr, size); - if (--depth_ < 0) return nullptr; + --depth_; return ptr; } From e84ac2d3beb36ed8a951e94c808aafb6c0baa13e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 Feb 2023 14:46:54 -0800 Subject: [PATCH 028/463] [ObjC] Run clang-format. Fix clang-format directive and reform file. No other changes. PiperOrigin-RevId: 509338775 --- objectivec/GPBMessage.m | 664 ++++++++++++++++++---------------------- 1 file changed, 297 insertions(+), 367 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 334217919c1e..f41457066a44 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -1821,7 +1821,7 @@ - (void)writeField:(GPBFieldDescriptor *)field toCodedOutputStream:(GPBCodedOutp //%PDDM-EXPAND-END (18 expansions) -// clang-format off + // clang-format on } } @@ -1848,8 +1848,7 @@ - (id)getExtension:(GPBExtensionDescriptor *)extension { value = [autocreatedExtensionMap_ objectForKey:extension]; if (!value) { // Auto create the message extensions to match normal fields. - value = CreateMessageWithAutocreatorForExtension(extension.msgClass, self, - extension); + value = CreateMessageWithAutocreatorForExtension(extension.msgClass, self, extension); if (autocreatedExtensionMap_ == nil) { autocreatedExtensionMap_ = [[NSMutableDictionary alloc] init]; @@ -1926,8 +1925,7 @@ - (void)setExtension:(GPBExtensionDescriptor *)extension value:(id)value { GPBExtensionDescriptor *descriptor = extension; if (GPBExtensionIsMessage(descriptor) && !descriptor.isRepeated) { - GPBMessage *autocreatedValue = - [[autocreatedExtensionMap_ objectForKey:extension] retain]; + GPBMessage *autocreatedValue = [[autocreatedExtensionMap_ objectForKey:extension] retain]; // Must remove from the map before calling GPBClearMessageAutocreator() so // that GPBClearMessageAutocreator() knows its safe to clear. [autocreatedExtensionMap_ removeObjectForKey:extension]; @@ -1959,9 +1957,7 @@ - (void)addExtension:(GPBExtensionDescriptor *)extension value:(id)value { GPBBecomeVisibleToAutocreator(self); } -- (void)setExtension:(GPBExtensionDescriptor *)extension - index:(NSUInteger)idx - value:(id)value { +- (void)setExtension:(GPBExtensionDescriptor *)extension index:(NSUInteger)idx value:(id)value { CheckExtension(self, extension); if (!extension.repeated) { @@ -1991,8 +1987,7 @@ - (void)clearExtension:(GPBExtensionDescriptor *)extension { #pragma mark - mergeFrom -- (void)mergeFromData:(NSData *)data - extensionRegistry:(id)extensionRegistry { +- (void)mergeFromData:(NSData *)data extensionRegistry:(id)extensionRegistry { GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; [input checkLastTagWas:0]; @@ -2024,35 +2019,30 @@ + (instancetype)parseFromData:(NSData *)data error:(NSError **)errorPtr { + (instancetype)parseFromData:(NSData *)data extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { - return [[[self alloc] initWithData:data - extensionRegistry:extensionRegistry + return [[[self alloc] initWithData:data extensionRegistry:extensionRegistry error:errorPtr] autorelease]; } + (instancetype)parseFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { - return - [[[self alloc] initWithCodedInputStream:input - extensionRegistry:extensionRegistry - error:errorPtr] autorelease]; + return [[[self alloc] initWithCodedInputStream:input + extensionRegistry:extensionRegistry + error:errorPtr] autorelease]; } #pragma mark - Parse Delimited From Data Support + (instancetype)parseDelimitedFromCodedInputStream:(GPBCodedInputStream *)input - extensionRegistry: - (id)extensionRegistry + extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { GPBMessage *message = [[[self alloc] init] autorelease]; @try { - [message mergeDelimitedFromCodedInputStream:input - extensionRegistry:extensionRegistry]; + [message mergeDelimitedFromCodedInputStream:input extensionRegistry:extensionRegistry]; if (errorPtr) { *errorPtr = nil; } - } - @catch (NSException *exception) { + } @catch (NSException *exception) { message = nil; if (errorPtr) { *errorPtr = ErrorFromException(exception); @@ -2098,12 +2088,10 @@ - (void)parseMessageSet:(GPBCodedInputStream *)input if (tag == GPBWireFormatMessageSetTypeIdTag) { typeId = GPBCodedInputStreamReadUInt32(state); if (typeId != 0) { - extension = [extensionRegistry extensionForDescriptor:[self descriptor] - fieldNumber:typeId]; + extension = [extensionRegistry extensionForDescriptor:[self descriptor] fieldNumber:typeId]; } } else if (tag == GPBWireFormatMessageSetMessageTag) { - rawBytes = - [GPBCodedInputStreamReadRetainedBytesNoCopy(state) autorelease]; + rawBytes = [GPBCodedInputStreamReadRetainedBytesNoCopy(state) autorelease]; } else { if (![input skipField:tag]) { break; @@ -2115,12 +2103,8 @@ - (void)parseMessageSet:(GPBCodedInputStream *)input if (rawBytes != nil && typeId != 0) { if (extension != nil) { - GPBCodedInputStream *newInput = - [[GPBCodedInputStream alloc] initWithData:rawBytes]; - ExtensionMergeFromInputStream(extension, - extension.packable, - newInput, - extensionRegistry, + GPBCodedInputStream *newInput = [[GPBCodedInputStream alloc] initWithData:rawBytes]; + ExtensionMergeFromInputStream(extension, extension.packable, newInput, extensionRegistry, self); [newInput release]; } else { @@ -2142,9 +2126,8 @@ - (BOOL)parseUnknownField:(GPBCodedInputStream *)input int32_t fieldNumber = GPBWireFormatGetTagFieldNumber(tag); GPBDescriptor *descriptor = [self descriptor]; - GPBExtensionDescriptor *extension = - [extensionRegistry extensionForDescriptor:descriptor - fieldNumber:fieldNumber]; + GPBExtensionDescriptor *extension = [extensionRegistry extensionForDescriptor:descriptor + fieldNumber:fieldNumber]; if (extension == nil) { if (descriptor.wireFormat && GPBWireFormatMessageSetItemTag == tag) { [self parseMessageSet:input extensionRegistry:extensionRegistry]; @@ -2152,23 +2135,14 @@ - (BOOL)parseUnknownField:(GPBCodedInputStream *)input } } else { if (extension.wireType == wireType) { - ExtensionMergeFromInputStream(extension, - extension.packable, - input, - extensionRegistry, - self); + ExtensionMergeFromInputStream(extension, extension.packable, input, extensionRegistry, self); return YES; } // Primitive, repeated types can be packed on unpacked on the wire, and are // parsed either way. - if ([extension isRepeated] && - !GPBDataTypeIsObject(extension->description_->dataType) && + if ([extension isRepeated] && !GPBDataTypeIsObject(extension->description_->dataType) && (extension.alternateWireType == wireType)) { - ExtensionMergeFromInputStream(extension, - !extension.packable, - input, - extensionRegistry, - self); + ExtensionMergeFromInputStream(extension, !extension.packable, input, extensionRegistry, self); return YES; } } @@ -2187,38 +2161,38 @@ - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data { #pragma mark - MergeFromCodedInputStream Support -static void MergeSingleFieldFromCodedInputStream( - GPBMessage *self, GPBFieldDescriptor *field, - GPBCodedInputStream *input, idextensionRegistry) { +static void MergeSingleFieldFromCodedInputStream(GPBMessage *self, GPBFieldDescriptor *field, + GPBCodedInputStream *input, + id extensionRegistry) { GPBDataType fieldDataType = GPBGetFieldDataType(field); switch (fieldDataType) { -#define CASE_SINGLE_POD(NAME, TYPE, FUNC_TYPE) \ - case GPBDataType##NAME: { \ - TYPE val = GPBCodedInputStreamRead##NAME(&input->state_); \ - GPBSet##FUNC_TYPE##IvarWithFieldPrivate(self, field, val); \ - break; \ - } -#define CASE_SINGLE_OBJECT(NAME) \ - case GPBDataType##NAME: { \ - id val = GPBCodedInputStreamReadRetained##NAME(&input->state_); \ - GPBSetRetainedObjectIvarWithFieldPrivate(self, field, val); \ - break; \ - } - CASE_SINGLE_POD(Bool, BOOL, Bool) - CASE_SINGLE_POD(Fixed32, uint32_t, UInt32) - CASE_SINGLE_POD(SFixed32, int32_t, Int32) - CASE_SINGLE_POD(Float, float, Float) - CASE_SINGLE_POD(Fixed64, uint64_t, UInt64) - CASE_SINGLE_POD(SFixed64, int64_t, Int64) - CASE_SINGLE_POD(Double, double, Double) - CASE_SINGLE_POD(Int32, int32_t, Int32) - CASE_SINGLE_POD(Int64, int64_t, Int64) - CASE_SINGLE_POD(SInt32, int32_t, Int32) - CASE_SINGLE_POD(SInt64, int64_t, Int64) - CASE_SINGLE_POD(UInt32, uint32_t, UInt32) - CASE_SINGLE_POD(UInt64, uint64_t, UInt64) - CASE_SINGLE_OBJECT(Bytes) - CASE_SINGLE_OBJECT(String) +#define CASE_SINGLE_POD(NAME, TYPE, FUNC_TYPE) \ + case GPBDataType##NAME: { \ + TYPE val = GPBCodedInputStreamRead##NAME(&input->state_); \ + GPBSet##FUNC_TYPE##IvarWithFieldPrivate(self, field, val); \ + break; \ + } +#define CASE_SINGLE_OBJECT(NAME) \ + case GPBDataType##NAME: { \ + id val = GPBCodedInputStreamReadRetained##NAME(&input->state_); \ + GPBSetRetainedObjectIvarWithFieldPrivate(self, field, val); \ + break; \ + } + CASE_SINGLE_POD(Bool, BOOL, Bool) + CASE_SINGLE_POD(Fixed32, uint32_t, UInt32) + CASE_SINGLE_POD(SFixed32, int32_t, Int32) + CASE_SINGLE_POD(Float, float, Float) + CASE_SINGLE_POD(Fixed64, uint64_t, UInt64) + CASE_SINGLE_POD(SFixed64, int64_t, Int64) + CASE_SINGLE_POD(Double, double, Double) + CASE_SINGLE_POD(Int32, int32_t, Int32) + CASE_SINGLE_POD(Int64, int64_t, Int64) + CASE_SINGLE_POD(SInt32, int32_t, Int32) + CASE_SINGLE_POD(SInt64, int64_t, Int64) + CASE_SINGLE_POD(UInt32, uint32_t, UInt32) + CASE_SINGLE_POD(UInt64, uint64_t, UInt64) + CASE_SINGLE_OBJECT(Bytes) + CASE_SINGLE_OBJECT(String) #undef CASE_SINGLE_POD #undef CASE_SINGLE_OBJECT @@ -2226,8 +2200,7 @@ static void MergeSingleFieldFromCodedInputStream( if (GPBGetHasIvarField(self, field)) { // GPBGetObjectIvarWithFieldNoAutocreate() avoids doing the has // check again. - GPBMessage *message = - GPBGetObjectIvarWithFieldNoAutocreate(self, field); + GPBMessage *message = GPBGetObjectIvarWithFieldNoAutocreate(self, field); [input readMessage:message extensionRegistry:extensionRegistry]; } else { GPBMessage *message = [[field.msgClass alloc] init]; @@ -2241,16 +2214,11 @@ static void MergeSingleFieldFromCodedInputStream( if (GPBGetHasIvarField(self, field)) { // GPBGetObjectIvarWithFieldNoAutocreate() avoids doing the has // check again. - GPBMessage *message = - GPBGetObjectIvarWithFieldNoAutocreate(self, field); - [input readGroup:GPBFieldNumber(field) - message:message - extensionRegistry:extensionRegistry]; + GPBMessage *message = GPBGetObjectIvarWithFieldNoAutocreate(self, field); + [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; } else { GPBMessage *message = [[field.msgClass alloc] init]; - [input readGroup:GPBFieldNumber(field) - message:message - extensionRegistry:extensionRegistry]; + [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; GPBSetRetainedObjectIvarWithFieldPrivate(self, field, message); } break; @@ -2268,9 +2236,9 @@ static void MergeSingleFieldFromCodedInputStream( } // switch } -static void MergeRepeatedPackedFieldFromCodedInputStream( - GPBMessage *self, GPBFieldDescriptor *field, - GPBCodedInputStream *input) { +static void MergeRepeatedPackedFieldFromCodedInputStream(GPBMessage *self, + GPBFieldDescriptor *field, + GPBCodedInputStream *input) { GPBDataType fieldDataType = GPBGetFieldDataType(field); GPBCodedInputStreamState *state = &input->state_; id genericArray = GetOrCreateArrayIvarWithField(self, field); @@ -2278,25 +2246,25 @@ static void MergeRepeatedPackedFieldFromCodedInputStream( size_t limit = GPBCodedInputStreamPushLimit(state, length); while (GPBCodedInputStreamBytesUntilLimit(state) > 0) { switch (fieldDataType) { -#define CASE_REPEATED_PACKED_POD(NAME, TYPE, ARRAY_TYPE) \ - case GPBDataType##NAME: { \ - TYPE val = GPBCodedInputStreamRead##NAME(state); \ - [(GPB##ARRAY_TYPE##Array *)genericArray addValue:val]; \ - break; \ - } - CASE_REPEATED_PACKED_POD(Bool, BOOL, Bool) - CASE_REPEATED_PACKED_POD(Fixed32, uint32_t, UInt32) - CASE_REPEATED_PACKED_POD(SFixed32, int32_t, Int32) - CASE_REPEATED_PACKED_POD(Float, float, Float) - CASE_REPEATED_PACKED_POD(Fixed64, uint64_t, UInt64) - CASE_REPEATED_PACKED_POD(SFixed64, int64_t, Int64) - CASE_REPEATED_PACKED_POD(Double, double, Double) - CASE_REPEATED_PACKED_POD(Int32, int32_t, Int32) - CASE_REPEATED_PACKED_POD(Int64, int64_t, Int64) - CASE_REPEATED_PACKED_POD(SInt32, int32_t, Int32) - CASE_REPEATED_PACKED_POD(SInt64, int64_t, Int64) - CASE_REPEATED_PACKED_POD(UInt32, uint32_t, UInt32) - CASE_REPEATED_PACKED_POD(UInt64, uint64_t, UInt64) +#define CASE_REPEATED_PACKED_POD(NAME, TYPE, ARRAY_TYPE) \ + case GPBDataType##NAME: { \ + TYPE val = GPBCodedInputStreamRead##NAME(state); \ + [(GPB##ARRAY_TYPE##Array *)genericArray addValue:val]; \ + break; \ + } + CASE_REPEATED_PACKED_POD(Bool, BOOL, Bool) + CASE_REPEATED_PACKED_POD(Fixed32, uint32_t, UInt32) + CASE_REPEATED_PACKED_POD(SFixed32, int32_t, Int32) + CASE_REPEATED_PACKED_POD(Float, float, Float) + CASE_REPEATED_PACKED_POD(Fixed64, uint64_t, UInt64) + CASE_REPEATED_PACKED_POD(SFixed64, int64_t, Int64) + CASE_REPEATED_PACKED_POD(Double, double, Double) + CASE_REPEATED_PACKED_POD(Int32, int32_t, Int32) + CASE_REPEATED_PACKED_POD(Int64, int64_t, Int64) + CASE_REPEATED_PACKED_POD(SInt32, int32_t, Int32) + CASE_REPEATED_PACKED_POD(SInt64, int64_t, Int64) + CASE_REPEATED_PACKED_POD(UInt32, uint32_t, UInt32) + CASE_REPEATED_PACKED_POD(UInt64, uint64_t, UInt64) #undef CASE_REPEATED_PACKED_POD case GPBDataTypeBytes: @@ -2309,7 +2277,7 @@ static void MergeRepeatedPackedFieldFromCodedInputStream( case GPBDataTypeEnum: { int32_t val = GPBCodedInputStreamReadEnum(state); if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { - [(GPBEnumArray*)genericArray addRawValue:val]; + [(GPBEnumArray *)genericArray addRawValue:val]; } else { GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; @@ -2317,66 +2285,64 @@ static void MergeRepeatedPackedFieldFromCodedInputStream( break; } } // switch - } // while(BytesUntilLimit() > 0) + } // while(BytesUntilLimit() > 0) GPBCodedInputStreamPopLimit(state, limit); } static void MergeRepeatedNotPackedFieldFromCodedInputStream( - GPBMessage *self, GPBFieldDescriptor *field, - GPBCodedInputStream *input, idextensionRegistry) { + GPBMessage *self, GPBFieldDescriptor *field, GPBCodedInputStream *input, + id extensionRegistry) { GPBCodedInputStreamState *state = &input->state_; id genericArray = GetOrCreateArrayIvarWithField(self, field); switch (GPBGetFieldDataType(field)) { #define CASE_REPEATED_NOT_PACKED_POD(NAME, TYPE, ARRAY_TYPE) \ - case GPBDataType##NAME: { \ - TYPE val = GPBCodedInputStreamRead##NAME(state); \ - [(GPB##ARRAY_TYPE##Array *)genericArray addValue:val]; \ - break; \ - } -#define CASE_REPEATED_NOT_PACKED_OBJECT(NAME) \ - case GPBDataType##NAME: { \ - id val = GPBCodedInputStreamReadRetained##NAME(state); \ - [(NSMutableArray*)genericArray addObject:val]; \ - [val release]; \ - break; \ - } - CASE_REPEATED_NOT_PACKED_POD(Bool, BOOL, Bool) - CASE_REPEATED_NOT_PACKED_POD(Fixed32, uint32_t, UInt32) - CASE_REPEATED_NOT_PACKED_POD(SFixed32, int32_t, Int32) - CASE_REPEATED_NOT_PACKED_POD(Float, float, Float) - CASE_REPEATED_NOT_PACKED_POD(Fixed64, uint64_t, UInt64) - CASE_REPEATED_NOT_PACKED_POD(SFixed64, int64_t, Int64) - CASE_REPEATED_NOT_PACKED_POD(Double, double, Double) - CASE_REPEATED_NOT_PACKED_POD(Int32, int32_t, Int32) - CASE_REPEATED_NOT_PACKED_POD(Int64, int64_t, Int64) - CASE_REPEATED_NOT_PACKED_POD(SInt32, int32_t, Int32) - CASE_REPEATED_NOT_PACKED_POD(SInt64, int64_t, Int64) - CASE_REPEATED_NOT_PACKED_POD(UInt32, uint32_t, UInt32) - CASE_REPEATED_NOT_PACKED_POD(UInt64, uint64_t, UInt64) - CASE_REPEATED_NOT_PACKED_OBJECT(Bytes) - CASE_REPEATED_NOT_PACKED_OBJECT(String) + case GPBDataType##NAME: { \ + TYPE val = GPBCodedInputStreamRead##NAME(state); \ + [(GPB##ARRAY_TYPE##Array *)genericArray addValue:val]; \ + break; \ + } +#define CASE_REPEATED_NOT_PACKED_OBJECT(NAME) \ + case GPBDataType##NAME: { \ + id val = GPBCodedInputStreamReadRetained##NAME(state); \ + [(NSMutableArray *)genericArray addObject:val]; \ + [val release]; \ + break; \ + } + CASE_REPEATED_NOT_PACKED_POD(Bool, BOOL, Bool) + CASE_REPEATED_NOT_PACKED_POD(Fixed32, uint32_t, UInt32) + CASE_REPEATED_NOT_PACKED_POD(SFixed32, int32_t, Int32) + CASE_REPEATED_NOT_PACKED_POD(Float, float, Float) + CASE_REPEATED_NOT_PACKED_POD(Fixed64, uint64_t, UInt64) + CASE_REPEATED_NOT_PACKED_POD(SFixed64, int64_t, Int64) + CASE_REPEATED_NOT_PACKED_POD(Double, double, Double) + CASE_REPEATED_NOT_PACKED_POD(Int32, int32_t, Int32) + CASE_REPEATED_NOT_PACKED_POD(Int64, int64_t, Int64) + CASE_REPEATED_NOT_PACKED_POD(SInt32, int32_t, Int32) + CASE_REPEATED_NOT_PACKED_POD(SInt64, int64_t, Int64) + CASE_REPEATED_NOT_PACKED_POD(UInt32, uint32_t, UInt32) + CASE_REPEATED_NOT_PACKED_POD(UInt64, uint64_t, UInt64) + CASE_REPEATED_NOT_PACKED_OBJECT(Bytes) + CASE_REPEATED_NOT_PACKED_OBJECT(String) #undef CASE_REPEATED_NOT_PACKED_POD #undef CASE_NOT_PACKED_OBJECT case GPBDataTypeMessage: { GPBMessage *message = [[field.msgClass alloc] init]; [input readMessage:message extensionRegistry:extensionRegistry]; - [(NSMutableArray*)genericArray addObject:message]; + [(NSMutableArray *)genericArray addObject:message]; [message release]; break; } case GPBDataTypeGroup: { GPBMessage *message = [[field.msgClass alloc] init]; - [input readGroup:GPBFieldNumber(field) - message:message - extensionRegistry:extensionRegistry]; - [(NSMutableArray*)genericArray addObject:message]; + [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; + [(NSMutableArray *)genericArray addObject:message]; [message release]; break; } case GPBDataTypeEnum: { int32_t val = GPBCodedInputStreamReadEnum(state); if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { - [(GPBEnumArray*)genericArray addRawValue:val]; + [(GPBEnumArray *)genericArray addRawValue:val]; } else { GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; @@ -2406,30 +2372,28 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input if (GPBFieldTag(fieldDescriptor) == tag) { GPBFieldType fieldType = fieldDescriptor.fieldType; if (fieldType == GPBFieldTypeSingle) { - MergeSingleFieldFromCodedInputStream(self, fieldDescriptor, - input, extensionRegistry); + MergeSingleFieldFromCodedInputStream(self, fieldDescriptor, input, extensionRegistry); // Well formed protos will only have a single field once, advance // the starting index to the next field. startingIndex += 1; } else if (fieldType == GPBFieldTypeRepeated) { if (fieldDescriptor.isPackable) { - MergeRepeatedPackedFieldFromCodedInputStream( - self, fieldDescriptor, input); + MergeRepeatedPackedFieldFromCodedInputStream(self, fieldDescriptor, input); // Well formed protos will only have a repeated field that is // packed once, advance the starting index to the next field. startingIndex += 1; } else { - MergeRepeatedNotPackedFieldFromCodedInputStream( - self, fieldDescriptor, input, extensionRegistry); + MergeRepeatedNotPackedFieldFromCodedInputStream(self, fieldDescriptor, input, + extensionRegistry); } } else { // fieldType == GPBFieldTypeMap // GPB*Dictionary or NSDictionary, exact type doesn't matter at this // point. id map = GetOrCreateMapIvarWithField(self, fieldDescriptor); [input readMapEntry:map - extensionRegistry:extensionRegistry - field:fieldDescriptor - parentMessage:self]; + extensionRegistry:extensionRegistry + field:fieldDescriptor + parentMessage:self]; } merged = YES; break; @@ -2450,14 +2414,13 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input (GPBFieldAlternateTag(fieldDescriptor) == tag)) { BOOL alternateIsPacked = !fieldDescriptor.isPackable; if (alternateIsPacked) { - MergeRepeatedPackedFieldFromCodedInputStream( - self, fieldDescriptor, input); + MergeRepeatedPackedFieldFromCodedInputStream(self, fieldDescriptor, input); // Well formed protos will only have a repeated field that is // packed once, advance the starting index to the next field. startingIndex += 1; } else { - MergeRepeatedNotPackedFieldFromCodedInputStream( - self, fieldDescriptor, input, extensionRegistry); + MergeRepeatedNotPackedFieldFromCodedInputStream(self, fieldDescriptor, input, + extensionRegistry); } merged = YES; break; @@ -2472,9 +2435,7 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input // zero signals EOF / limit reached return; } else { - if (![self parseUnknownField:input - extensionRegistry:extensionRegistry - tag:tag]) { + if (![self parseUnknownField:input extensionRegistry:extensionRegistry tag:tag]) { // it's an endgroup tag return; } @@ -2489,8 +2450,7 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input - (void)mergeFrom:(GPBMessage *)other { Class selfClass = [self class]; Class otherClass = [other class]; - if (!([selfClass isSubclassOfClass:otherClass] || - [otherClass isSubclassOfClass:selfClass])) { + if (!([selfClass isSubclassOfClass:otherClass] || [otherClass isSubclassOfClass:selfClass])) { [NSException raise:NSInvalidArgumentException format:@"Classes must match %@ != %@", selfClass, otherClass]; } @@ -2512,39 +2472,32 @@ - (void)mergeFrom:(GPBMessage *)other { GPBDataType fieldDataType = GPBGetFieldDataType(field); switch (fieldDataType) { case GPBDataTypeBool: - GPBSetBoolIvarWithFieldPrivate( - self, field, GPBGetMessageBoolField(other, field)); + GPBSetBoolIvarWithFieldPrivate(self, field, GPBGetMessageBoolField(other, field)); break; case GPBDataTypeSFixed32: case GPBDataTypeEnum: case GPBDataTypeInt32: case GPBDataTypeSInt32: - GPBSetInt32IvarWithFieldPrivate( - self, field, GPBGetMessageInt32Field(other, field)); + GPBSetInt32IvarWithFieldPrivate(self, field, GPBGetMessageInt32Field(other, field)); break; case GPBDataTypeFixed32: case GPBDataTypeUInt32: - GPBSetUInt32IvarWithFieldPrivate( - self, field, GPBGetMessageUInt32Field(other, field)); + GPBSetUInt32IvarWithFieldPrivate(self, field, GPBGetMessageUInt32Field(other, field)); break; case GPBDataTypeSFixed64: case GPBDataTypeInt64: case GPBDataTypeSInt64: - GPBSetInt64IvarWithFieldPrivate( - self, field, GPBGetMessageInt64Field(other, field)); + GPBSetInt64IvarWithFieldPrivate(self, field, GPBGetMessageInt64Field(other, field)); break; case GPBDataTypeFixed64: case GPBDataTypeUInt64: - GPBSetUInt64IvarWithFieldPrivate( - self, field, GPBGetMessageUInt64Field(other, field)); + GPBSetUInt64IvarWithFieldPrivate(self, field, GPBGetMessageUInt64Field(other, field)); break; case GPBDataTypeFloat: - GPBSetFloatIvarWithFieldPrivate( - self, field, GPBGetMessageFloatField(other, field)); + GPBSetFloatIvarWithFieldPrivate(self, field, GPBGetMessageFloatField(other, field)); break; case GPBDataTypeDouble: - GPBSetDoubleIvarWithFieldPrivate( - self, field, GPBGetMessageDoubleField(other, field)); + GPBSetDoubleIvarWithFieldPrivate(self, field, GPBGetMessageDoubleField(other, field)); break; case GPBDataTypeBytes: case GPBDataTypeString: { @@ -2556,8 +2509,7 @@ - (void)mergeFrom:(GPBMessage *)other { case GPBDataTypeGroup: { id otherVal = GPBGetObjectIvarWithFieldNoAutocreate(other, field); if (GPBGetHasIvar(self, hasIndex, fieldNumber)) { - GPBMessage *message = - GPBGetObjectIvarWithFieldNoAutocreate(self, field); + GPBMessage *message = GPBGetObjectIvarWithFieldNoAutocreate(self, field); [message mergeFrom:otherVal]; } else { GPBMessage *message = [otherVal copy]; @@ -2565,27 +2517,23 @@ - (void)mergeFrom:(GPBMessage *)other { } break; } - } // switch() + } // switch() } else if (fieldType == GPBFieldTypeRepeated) { // In the case of a list, they need to be appended, and there is no // _hasIvar to worry about setting. - id otherArray = - GPBGetObjectIvarWithFieldNoAutocreate(other, field); + id otherArray = GPBGetObjectIvarWithFieldNoAutocreate(other, field); if (otherArray) { GPBDataType fieldDataType = field->description_->dataType; if (GPBDataTypeIsObject(fieldDataType)) { - NSMutableArray *resultArray = - GetOrCreateArrayIvarWithField(self, field); + NSMutableArray *resultArray = GetOrCreateArrayIvarWithField(self, field); [resultArray addObjectsFromArray:otherArray]; } else if (fieldDataType == GPBDataTypeEnum) { - GPBEnumArray *resultArray = - GetOrCreateArrayIvarWithField(self, field); + GPBEnumArray *resultArray = GetOrCreateArrayIvarWithField(self, field); [resultArray addRawValuesFromArray:otherArray]; } else { // The array type doesn't matter, that all implement // -addValuesFromArray:. - GPBInt32Array *resultArray = - GetOrCreateArrayIvarWithField(self, field); + GPBInt32Array *resultArray = GetOrCreateArrayIvarWithField(self, field); [resultArray addValuesFromArray:otherArray]; } } @@ -2596,27 +2544,23 @@ - (void)mergeFrom:(GPBMessage *)other { if (otherDict) { GPBDataType keyDataType = field.mapKeyDataType; GPBDataType valueDataType = field->description_->dataType; - if (GPBDataTypeIsObject(keyDataType) && - GPBDataTypeIsObject(valueDataType)) { - NSMutableDictionary *resultDict = - GetOrCreateMapIvarWithField(self, field); + if (GPBDataTypeIsObject(keyDataType) && GPBDataTypeIsObject(valueDataType)) { + NSMutableDictionary *resultDict = GetOrCreateMapIvarWithField(self, field); [resultDict addEntriesFromDictionary:otherDict]; } else if (valueDataType == GPBDataTypeEnum) { // The exact type doesn't matter, just need to know it is a // GPB*EnumDictionary. - GPBInt32EnumDictionary *resultDict = - GetOrCreateMapIvarWithField(self, field); + GPBInt32EnumDictionary *resultDict = GetOrCreateMapIvarWithField(self, field); [resultDict addRawEntriesFromDictionary:otherDict]; } else { // The exact type doesn't matter, they all implement // -addEntriesFromDictionary:. - GPBInt32Int32Dictionary *resultDict = - GetOrCreateMapIvarWithField(self, field); + GPBInt32Int32Dictionary *resultDict = GetOrCreateMapIvarWithField(self, field); [resultDict addEntriesFromDictionary:otherDict]; } } } // if (fieldType)..else if...else - } // for(fields) + } // for(fields) // Unknown fields. if (!unknownFields_) { @@ -2632,8 +2576,7 @@ - (void)mergeFrom:(GPBMessage *)other { } if (extensionMap_ == nil) { - extensionMap_ = - CloneExtensionMap(other->extensionMap_, NSZoneFromPointer(self)); + extensionMap_ = CloneExtensionMap(other->extensionMap_, NSZoneFromPointer(self)); } else { for (GPBExtensionDescriptor *extension in other->extensionMap_) { id otherValue = [other->extensionMap_ objectForKey:extension]; @@ -2671,8 +2614,7 @@ - (void)mergeFrom:(GPBMessage *)other { } if (isMessageExtension && !extension.isRepeated) { - GPBMessage *autocreatedValue = - [[autocreatedExtensionMap_ objectForKey:extension] retain]; + GPBMessage *autocreatedValue = [[autocreatedExtensionMap_ objectForKey:extension] retain]; // Must remove from the map before calling GPBClearMessageAutocreator() // so that GPBClearMessageAutocreator() knows its safe to clear. [autocreatedExtensionMap_ removeObjectForKey:extension]; @@ -2706,10 +2648,8 @@ - (BOOL)isEqual:(id)other { // NOTE: These are NSArray/GPB*Array or NSDictionary/GPB*Dictionary, but // the type doesn't really matter as the objects all support -count and // -isEqual:. - NSArray *resultMapOrArray = - GPBGetObjectIvarWithFieldNoAutocreate(self, field); - NSArray *otherMapOrArray = - GPBGetObjectIvarWithFieldNoAutocreate(other, field); + NSArray *resultMapOrArray = GPBGetObjectIvarWithFieldNoAutocreate(self, field); + NSArray *otherMapOrArray = GPBGetObjectIvarWithFieldNoAutocreate(other, field); // nil and empty are equal if (resultMapOrArray.count != 0 || otherMapOrArray.count != 0) { if (![resultMapOrArray isEqual:otherMapOrArray]) { @@ -2787,9 +2727,9 @@ - (BOOL)isEqual:(id)other { } break; } - } // switch() - } // if(mapOrArray)...else - } // for(fields) + } // switch() + } // if(mapOrArray)...else + } // for(fields) // nil and empty are equal if (extensionMap_.count != 0 || otherMsg->extensionMap_.count != 0) { @@ -2800,8 +2740,7 @@ - (BOOL)isEqual:(id)other { // nil and empty are equal GPBUnknownFieldSet *otherUnknowns = otherMsg->unknownFields_; - if ([unknownFields_ countOfFields] != 0 || - [otherUnknowns countOfFields] != 0) { + if ([unknownFields_ countOfFields] != 0 || [otherUnknowns countOfFields] != 0) { if (![unknownFields_ isEqual:otherUnknowns]) { return NO; } @@ -2904,7 +2843,7 @@ - (NSUInteger)hash { result = prime * result + (NSUInteger)[[*valPtr class] descriptor]; break; } - } // switch() + } // switch() } } @@ -2917,8 +2856,8 @@ - (NSUInteger)hash { - (NSString *)description { NSString *textFormat = GPBTextFormatForMessage(self, @" "); - NSString *description = [NSString - stringWithFormat:@"<%@ %p>: {\n%@}", [self class], self, textFormat]; + NSString *description = + [NSString stringWithFormat:@"<%@ %p>: {\n%@}", [self class], self, textFormat]; return description; } @@ -2956,44 +2895,43 @@ - (size_t)serializedSize { uint32_t fieldNumber = GPBFieldNumber(fieldDescriptor); switch (fieldDataType) { -#define CASE_SINGLE_POD(NAME, TYPE, FUNC_TYPE) \ - case GPBDataType##NAME: { \ - TYPE fieldVal = GPBGetMessage##FUNC_TYPE##Field(self, fieldDescriptor); \ - result += GPBCompute##NAME##Size(fieldNumber, fieldVal); \ - break; \ - } -#define CASE_SINGLE_OBJECT(NAME) \ - case GPBDataType##NAME: { \ - id fieldVal = GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); \ - result += GPBCompute##NAME##Size(fieldNumber, fieldVal); \ - break; \ - } - CASE_SINGLE_POD(Bool, BOOL, Bool) - CASE_SINGLE_POD(Fixed32, uint32_t, UInt32) - CASE_SINGLE_POD(SFixed32, int32_t, Int32) - CASE_SINGLE_POD(Float, float, Float) - CASE_SINGLE_POD(Fixed64, uint64_t, UInt64) - CASE_SINGLE_POD(SFixed64, int64_t, Int64) - CASE_SINGLE_POD(Double, double, Double) - CASE_SINGLE_POD(Int32, int32_t, Int32) - CASE_SINGLE_POD(Int64, int64_t, Int64) - CASE_SINGLE_POD(SInt32, int32_t, Int32) - CASE_SINGLE_POD(SInt64, int64_t, Int64) - CASE_SINGLE_POD(UInt32, uint32_t, UInt32) - CASE_SINGLE_POD(UInt64, uint64_t, UInt64) - CASE_SINGLE_OBJECT(Bytes) - CASE_SINGLE_OBJECT(String) - CASE_SINGLE_OBJECT(Message) - CASE_SINGLE_OBJECT(Group) - CASE_SINGLE_POD(Enum, int32_t, Int32) +#define CASE_SINGLE_POD(NAME, TYPE, FUNC_TYPE) \ + case GPBDataType##NAME: { \ + TYPE fieldVal = GPBGetMessage##FUNC_TYPE##Field(self, fieldDescriptor); \ + result += GPBCompute##NAME##Size(fieldNumber, fieldVal); \ + break; \ + } +#define CASE_SINGLE_OBJECT(NAME) \ + case GPBDataType##NAME: { \ + id fieldVal = GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); \ + result += GPBCompute##NAME##Size(fieldNumber, fieldVal); \ + break; \ + } + CASE_SINGLE_POD(Bool, BOOL, Bool) + CASE_SINGLE_POD(Fixed32, uint32_t, UInt32) + CASE_SINGLE_POD(SFixed32, int32_t, Int32) + CASE_SINGLE_POD(Float, float, Float) + CASE_SINGLE_POD(Fixed64, uint64_t, UInt64) + CASE_SINGLE_POD(SFixed64, int64_t, Int64) + CASE_SINGLE_POD(Double, double, Double) + CASE_SINGLE_POD(Int32, int32_t, Int32) + CASE_SINGLE_POD(Int64, int64_t, Int64) + CASE_SINGLE_POD(SInt32, int32_t, Int32) + CASE_SINGLE_POD(SInt64, int64_t, Int64) + CASE_SINGLE_POD(UInt32, uint32_t, UInt32) + CASE_SINGLE_POD(UInt64, uint64_t, UInt64) + CASE_SINGLE_OBJECT(Bytes) + CASE_SINGLE_OBJECT(String) + CASE_SINGLE_OBJECT(Message) + CASE_SINGLE_OBJECT(Group) + CASE_SINGLE_POD(Enum, int32_t, Int32) #undef CASE_SINGLE_POD #undef CASE_SINGLE_OBJECT } - // Repeated Fields + // Repeated Fields } else if (fieldType == GPBFieldTypeRepeated) { - id genericArray = - GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); + id genericArray = GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); NSUInteger count = [genericArray count]; if (count == 0) { continue; // Nothing to add. @@ -3001,41 +2939,41 @@ - (size_t)serializedSize { __block size_t dataSize = 0; switch (fieldDataType) { -#define CASE_REPEATED_POD(NAME, TYPE, ARRAY_TYPE) \ - CASE_REPEATED_POD_EXTRA(NAME, TYPE, ARRAY_TYPE, ) -#define CASE_REPEATED_POD_EXTRA(NAME, TYPE, ARRAY_TYPE, ARRAY_ACCESSOR_NAME) \ - case GPBDataType##NAME: { \ - GPB##ARRAY_TYPE##Array *array = genericArray; \ - [array enumerate##ARRAY_ACCESSOR_NAME##ValuesWithBlock:^(TYPE value, __unused NSUInteger idx, __unused BOOL *stop) { \ - dataSize += GPBCompute##NAME##SizeNoTag(value); \ - }]; \ - break; \ - } -#define CASE_REPEATED_OBJECT(NAME) \ - case GPBDataType##NAME: { \ - for (id value in genericArray) { \ - dataSize += GPBCompute##NAME##SizeNoTag(value); \ - } \ - break; \ - } - CASE_REPEATED_POD(Bool, BOOL, Bool) - CASE_REPEATED_POD(Fixed32, uint32_t, UInt32) - CASE_REPEATED_POD(SFixed32, int32_t, Int32) - CASE_REPEATED_POD(Float, float, Float) - CASE_REPEATED_POD(Fixed64, uint64_t, UInt64) - CASE_REPEATED_POD(SFixed64, int64_t, Int64) - CASE_REPEATED_POD(Double, double, Double) - CASE_REPEATED_POD(Int32, int32_t, Int32) - CASE_REPEATED_POD(Int64, int64_t, Int64) - CASE_REPEATED_POD(SInt32, int32_t, Int32) - CASE_REPEATED_POD(SInt64, int64_t, Int64) - CASE_REPEATED_POD(UInt32, uint32_t, UInt32) - CASE_REPEATED_POD(UInt64, uint64_t, UInt64) - CASE_REPEATED_OBJECT(Bytes) - CASE_REPEATED_OBJECT(String) - CASE_REPEATED_OBJECT(Message) - CASE_REPEATED_OBJECT(Group) - CASE_REPEATED_POD_EXTRA(Enum, int32_t, Enum, Raw) +#define CASE_REPEATED_POD(NAME, TYPE, ARRAY_TYPE) CASE_REPEATED_POD_EXTRA(NAME, TYPE, ARRAY_TYPE, ) +#define CASE_REPEATED_POD_EXTRA(NAME, TYPE, ARRAY_TYPE, ARRAY_ACCESSOR_NAME) \ + case GPBDataType##NAME: { \ + GPB##ARRAY_TYPE##Array *array = genericArray; \ + [array enumerate##ARRAY_ACCESSOR_NAME## \ + ValuesWithBlock:^(TYPE value, __unused NSUInteger idx, __unused BOOL * stop) { \ + dataSize += GPBCompute##NAME##SizeNoTag(value); \ + }]; \ + break; \ + } +#define CASE_REPEATED_OBJECT(NAME) \ + case GPBDataType##NAME: { \ + for (id value in genericArray) { \ + dataSize += GPBCompute##NAME##SizeNoTag(value); \ + } \ + break; \ + } + CASE_REPEATED_POD(Bool, BOOL, Bool) + CASE_REPEATED_POD(Fixed32, uint32_t, UInt32) + CASE_REPEATED_POD(SFixed32, int32_t, Int32) + CASE_REPEATED_POD(Float, float, Float) + CASE_REPEATED_POD(Fixed64, uint64_t, UInt64) + CASE_REPEATED_POD(SFixed64, int64_t, Int64) + CASE_REPEATED_POD(Double, double, Double) + CASE_REPEATED_POD(Int32, int32_t, Int32) + CASE_REPEATED_POD(Int64, int64_t, Int64) + CASE_REPEATED_POD(SInt32, int32_t, Int32) + CASE_REPEATED_POD(SInt64, int64_t, Int64) + CASE_REPEATED_POD(UInt32, uint32_t, UInt32) + CASE_REPEATED_POD(UInt64, uint64_t, UInt64) + CASE_REPEATED_OBJECT(Bytes) + CASE_REPEATED_OBJECT(String) + CASE_REPEATED_OBJECT(Message) + CASE_REPEATED_OBJECT(Group) + CASE_REPEATED_POD_EXTRA(Enum, int32_t, Enum, Raw) #undef CASE_REPEATED_POD #undef CASE_REPEATED_POD_EXTRA #undef CASE_REPEATED_OBJECT @@ -3053,20 +2991,18 @@ - (size_t)serializedSize { result += count * tagSize; } - // Map<> Fields + // Map<> Fields } else { // fieldType == GPBFieldTypeMap if (GPBDataTypeIsObject(fieldDataType) && (fieldDescriptor.mapKeyDataType == GPBDataTypeString)) { // If key type was string, then the map is an NSDictionary. - NSDictionary *map = - GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); + NSDictionary *map = GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); if (map) { result += GPBDictionaryComputeSizeInternalHelper(map, fieldDescriptor); } } else { // Type will be GPB*GroupDictionary, exact type doesn't matter. - GPBInt32Int32Dictionary *map = - GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); + GPBInt32Int32Dictionary *map = GPBGetObjectIvarWithFieldNoAutocreate(self, fieldDescriptor); result += [map computeSerializedSizeAsField:fieldDescriptor]; } } @@ -3102,40 +3038,40 @@ static void ResolveIvarGet(__unsafe_unretained GPBFieldDescriptor *field, ResolveIvarAccessorMethodResult *result) { GPBDataType fieldDataType = GPBGetFieldDataType(field); switch (fieldDataType) { -#define CASE_GET(NAME, TYPE, TRUE_NAME) \ - case GPBDataType##NAME: { \ - result->impToAdd = imp_implementationWithBlock(^(id obj) { \ - return GPBGetMessage##TRUE_NAME##Field(obj, field); \ - }); \ - result->encodingSelector = @selector(get##NAME); \ - break; \ - } -#define CASE_GET_OBJECT(NAME, TYPE, TRUE_NAME) \ - case GPBDataType##NAME: { \ - result->impToAdd = imp_implementationWithBlock(^(id obj) { \ - return GPBGetObjectIvarWithField(obj, field); \ - }); \ - result->encodingSelector = @selector(get##NAME); \ - break; \ - } - CASE_GET(Bool, BOOL, Bool) - CASE_GET(Fixed32, uint32_t, UInt32) - CASE_GET(SFixed32, int32_t, Int32) - CASE_GET(Float, float, Float) - CASE_GET(Fixed64, uint64_t, UInt64) - CASE_GET(SFixed64, int64_t, Int64) - CASE_GET(Double, double, Double) - CASE_GET(Int32, int32_t, Int32) - CASE_GET(Int64, int64_t, Int64) - CASE_GET(SInt32, int32_t, Int32) - CASE_GET(SInt64, int64_t, Int64) - CASE_GET(UInt32, uint32_t, UInt32) - CASE_GET(UInt64, uint64_t, UInt64) - CASE_GET_OBJECT(Bytes, id, Object) - CASE_GET_OBJECT(String, id, Object) - CASE_GET_OBJECT(Message, id, Object) - CASE_GET_OBJECT(Group, id, Object) - CASE_GET(Enum, int32_t, Enum) +#define CASE_GET(NAME, TYPE, TRUE_NAME) \ + case GPBDataType##NAME: { \ + result->impToAdd = imp_implementationWithBlock(^(id obj) { \ + return GPBGetMessage##TRUE_NAME##Field(obj, field); \ + }); \ + result->encodingSelector = @selector(get##NAME); \ + break; \ + } +#define CASE_GET_OBJECT(NAME, TYPE, TRUE_NAME) \ + case GPBDataType##NAME: { \ + result->impToAdd = imp_implementationWithBlock(^(id obj) { \ + return GPBGetObjectIvarWithField(obj, field); \ + }); \ + result->encodingSelector = @selector(get##NAME); \ + break; \ + } + CASE_GET(Bool, BOOL, Bool) + CASE_GET(Fixed32, uint32_t, UInt32) + CASE_GET(SFixed32, int32_t, Int32) + CASE_GET(Float, float, Float) + CASE_GET(Fixed64, uint64_t, UInt64) + CASE_GET(SFixed64, int64_t, Int64) + CASE_GET(Double, double, Double) + CASE_GET(Int32, int32_t, Int32) + CASE_GET(Int64, int64_t, Int64) + CASE_GET(SInt32, int32_t, Int32) + CASE_GET(SInt64, int64_t, Int64) + CASE_GET(UInt32, uint32_t, UInt32) + CASE_GET(UInt64, uint64_t, UInt64) + CASE_GET_OBJECT(Bytes, id, Object) + CASE_GET_OBJECT(String, id, Object) + CASE_GET_OBJECT(Message, id, Object) + CASE_GET_OBJECT(Group, id, Object) + CASE_GET(Enum, int32_t, Enum) #undef CASE_GET } } @@ -3145,40 +3081,40 @@ static void ResolveIvarSet(__unsafe_unretained GPBFieldDescriptor *field, ResolveIvarAccessorMethodResult *result) { GPBDataType fieldDataType = GPBGetFieldDataType(field); switch (fieldDataType) { -#define CASE_SET(NAME, TYPE, TRUE_NAME) \ - case GPBDataType##NAME: { \ - result->impToAdd = imp_implementationWithBlock(^(id obj, TYPE value) { \ - return GPBSet##TRUE_NAME##IvarWithFieldPrivate(obj, field, value); \ - }); \ - result->encodingSelector = @selector(set##NAME:); \ - break; \ - } -#define CASE_SET_COPY(NAME) \ - case GPBDataType##NAME: { \ - result->impToAdd = imp_implementationWithBlock(^(id obj, id value) { \ - return GPBSetRetainedObjectIvarWithFieldPrivate(obj, field, [value copy]); \ - }); \ - result->encodingSelector = @selector(set##NAME:); \ - break; \ - } - CASE_SET(Bool, BOOL, Bool) - CASE_SET(Fixed32, uint32_t, UInt32) - CASE_SET(SFixed32, int32_t, Int32) - CASE_SET(Float, float, Float) - CASE_SET(Fixed64, uint64_t, UInt64) - CASE_SET(SFixed64, int64_t, Int64) - CASE_SET(Double, double, Double) - CASE_SET(Int32, int32_t, Int32) - CASE_SET(Int64, int64_t, Int64) - CASE_SET(SInt32, int32_t, Int32) - CASE_SET(SInt64, int64_t, Int64) - CASE_SET(UInt32, uint32_t, UInt32) - CASE_SET(UInt64, uint64_t, UInt64) - CASE_SET_COPY(Bytes) - CASE_SET_COPY(String) - CASE_SET(Message, id, Object) - CASE_SET(Group, id, Object) - CASE_SET(Enum, int32_t, Enum) +#define CASE_SET(NAME, TYPE, TRUE_NAME) \ + case GPBDataType##NAME: { \ + result->impToAdd = imp_implementationWithBlock(^(id obj, TYPE value) { \ + return GPBSet##TRUE_NAME##IvarWithFieldPrivate(obj, field, value); \ + }); \ + result->encodingSelector = @selector(set##NAME:); \ + break; \ + } +#define CASE_SET_COPY(NAME) \ + case GPBDataType##NAME: { \ + result->impToAdd = imp_implementationWithBlock(^(id obj, id value) { \ + return GPBSetRetainedObjectIvarWithFieldPrivate(obj, field, [value copy]); \ + }); \ + result->encodingSelector = @selector(set##NAME:); \ + break; \ + } + CASE_SET(Bool, BOOL, Bool) + CASE_SET(Fixed32, uint32_t, UInt32) + CASE_SET(SFixed32, int32_t, Int32) + CASE_SET(Float, float, Float) + CASE_SET(Fixed64, uint64_t, UInt64) + CASE_SET(SFixed64, int64_t, Int64) + CASE_SET(Double, double, Double) + CASE_SET(Int32, int32_t, Int32) + CASE_SET(Int64, int64_t, Int64) + CASE_SET(SInt32, int32_t, Int32) + CASE_SET(SInt64, int64_t, Int64) + CASE_SET(UInt32, uint32_t, UInt32) + CASE_SET(UInt64, uint64_t, UInt64) + CASE_SET_COPY(Bytes) + CASE_SET_COPY(String) + CASE_SET(Message, id, Object) + CASE_SET(Group, id, Object) + CASE_SET(Enum, int32_t, Enum) #undef CASE_SET } } @@ -3217,8 +3153,7 @@ + (BOOL)resolveInstanceMethod:(SEL)sel { result.impToAdd = imp_implementationWithBlock(^(id obj, BOOL value) { if (value) { [NSException raise:NSInvalidArgumentException - format:@"%@: %@ can only be set to NO (to clear field).", - [obj class], + format:@"%@: %@ can only be set to NO (to clear field).", [obj class], NSStringFromSelector(field->setHasSel_)]; } GPBClearMessageField(obj, field); @@ -3262,8 +3197,7 @@ + (BOOL)resolveInstanceMethod:(SEL)sel { result.impToAdd = imp_implementationWithBlock(^(id obj) { // Type doesn't matter, all *Array and *Dictionary types support // -count. - NSArray *arrayOrMap = - GPBGetObjectIvarWithFieldNoAutocreate(obj, field); + NSArray *arrayOrMap = GPBGetObjectIvarWithFieldNoAutocreate(obj, field); return [arrayOrMap count]; }); result.encodingSelector = @selector(getArrayCount); @@ -3272,8 +3206,7 @@ + (BOOL)resolveInstanceMethod:(SEL)sel { } } if (result.impToAdd) { - const char *encoding = - GPBMessageEncodingForSelector(result.encodingSelector, YES); + const char *encoding = GPBMessageEncodingForSelector(result.encodingSelector, YES); Class msgClass = descriptor.messageClass; BOOL methodAdded = class_addMethod(msgClass, sel, result.impToAdd, encoding); // class_addMethod() is documented as also failing if the method was already @@ -3307,8 +3240,7 @@ + (BOOL)supportsSecureCoding { - (instancetype)initWithCoder:(NSCoder *)aDecoder { self = [self init]; if (self) { - NSData *data = - [aDecoder decodeObjectOfClass:[NSData class] forKey:kGPBDataCoderKey]; + NSData *data = [aDecoder decodeObjectOfClass:[NSData class] forKey:kGPBDataCoderKey]; if (data.length) { [self mergeFromData:data extensionRegistry:nil]; } @@ -3355,8 +3287,7 @@ id GPBGetMessageRepeatedField(GPBMessage *self, GPBFieldDescriptor *field) { #if defined(DEBUG) && DEBUG if (field.fieldType != GPBFieldTypeRepeated) { [NSException raise:NSInvalidArgumentException - format:@"%@.%@ is not a repeated field.", - [self class], field.name]; + format:@"%@.%@ is not a repeated field.", [self class], field.name]; } #endif return GetOrCreateArrayIvarWithField(self, field); @@ -3367,8 +3298,7 @@ id GPBGetMessageMapField(GPBMessage *self, GPBFieldDescriptor *field) { #if defined(DEBUG) && DEBUG if (field.fieldType != GPBFieldTypeMap) { [NSException raise:NSInvalidArgumentException - format:@"%@.%@ is not a map<> field.", - [self class], field.name]; + format:@"%@.%@ is not a map<> field.", [self class], field.name]; } #endif return GetOrCreateMapIvarWithField(self, field); From 35e89279bc699401d6b6ecec529b912118e18792 Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Mon, 13 Feb 2023 16:06:20 -0800 Subject: [PATCH 029/463] Work around errornous warning in gcc <= 12.2 gcc 12.[12] emits a -Wrestrict warning if compiled with -O2/-O3 and -std=c++20 enabled when assigning a string with a single character to a std::string. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105329 This is benign; disable this broken warning for up to 12.4. PiperOrigin-RevId: 509358966 --- src/google/protobuf/port_def.inc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index d87b6b992906..3283030c16dd 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -1027,6 +1027,13 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // int index = ... // int value = vec[index]; #pragma GCC diagnostic ignored "-Wsign-conversion" +#if __GNUC__ == 12 && __GNUC_MINOR__ < 4 +// Wrong warning emitted when assigning a single char c-string to a std::string +// in c++20 mode and optimization on. +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105329 +// Planned to be fixed by 12.3 but widen window to 12.4. +#pragma GCC diagnostic ignored "-Wrestrict" +#endif #endif // __GNUC__ // Silence some MSVC warnings in all our code. From d447b2924f695ffa55f6bdfc3537b97bc12d9504 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 13 Feb 2023 20:22:21 -0800 Subject: [PATCH 030/463] Run continuous tests hourly to boost statistics for analysis PiperOrigin-RevId: 509409873 --- .github/workflows/test_runner.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index 9318af007752..ce3268a21249 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -12,8 +12,9 @@ name: Tests on: # continuous schedule: - # Run daily at 10 AM UTC (2 AM PDT) - - cron: 0 10 * * * + # TODO(mkruskal) Run daily at 10 AM UTC (2 AM PDT) + # Run every hour for now to gather statistics + - cron: 0 * * * * # postsubmit push: From 6e327701e9f3e6b7fb03d86e6dbe7531442eb292 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 Feb 2023 08:31:01 -0800 Subject: [PATCH 031/463] Automated rollback of commit 3493da38781eecfd08676a7a29d7294ad480179a. PiperOrigin-RevId: 509537125 --- .../src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs index 78008acd802d..1fb08db5edd0 100644 --- a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs +++ b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs @@ -397,6 +397,11 @@ public void DescriptorProtoFileDescriptor() [Test] public void DescriptorImportingExtensionsFromOldCodeGen() { + if (MethodOptions.Descriptor.FullName != "google.protobuf.MethodOptions") + { + Assert.Ignore("Embedded descriptor for OldExtensions expects google.protobuf reflection package."); + } + // The extension collection includes a null extension. There's not a lot we can do about that // in itself, as the old generator didn't provide us the extension information. var extensions = TestProtos.OldGenerator.OldExtensions2Reflection.Descriptor.Extensions; From 7ee157bc356ecb4a528692d735c2e73aaad21b43 Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Tue, 14 Feb 2023 08:54:52 -0800 Subject: [PATCH 032/463] Generate C# conformance test in script (#11880) This is already generated code, but wasn't included in our generation script. This change makes sure it will be up-to-date, and uses the now-conventional .pb.cs file extension. This does *not* affect the conformance tests run from Bazel, as they generate the file separately and build the conformance tests using only Program.cs and the freshly-generated code. However, the csharp/src/Google.Protobuf.Conformance/BUILD.bazel still needs to be kept up-to-date with the filename change. Closes #11880 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11880 from jskeet:generate-conformance 34bb22cc46cb67b58c302ca77541b45035815229 PiperOrigin-RevId: 509542860 --- csharp/generate_protos.sh | 5 + .../Google.Protobuf.Conformance/BUILD.bazel | 4 +- .../Conformance.cs | 1590 ----------------- 3 files changed, 7 insertions(+), 1592 deletions(-) delete mode 100644 csharp/src/Google.Protobuf.Conformance/Conformance.cs diff --git a/csharp/generate_protos.sh b/csharp/generate_protos.sh index cd682c2a89d3..17b1640cb932 100755 --- a/csharp/generate_protos.sh +++ b/csharp/generate_protos.sh @@ -79,3 +79,8 @@ $PROTOC -Isrc -I. \ $PROTOC -Iexamples -Isrc --csharp_out=csharp/src/AddressBook \ --csharp_opt=file_extension=.pb.cs \ examples/addressbook.proto + +# Conformance tests +$PROTOC -I. --csharp_out=csharp/src/Google.Protobuf.Conformance \ + --csharp_opt=file_extension=.pb.cs \ + conformance/conformance.proto diff --git a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel index 12ad0f7953f7..358c44c10888 100644 --- a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel +++ b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel @@ -37,7 +37,7 @@ CONFORMANCE_DEPS = [ filegroup( name = "srcs", srcs = [ - "Conformance.cs", + "Conformance.pb.cs", "Program.cs", "Google.Protobuf.Conformance.csproj", ], @@ -88,7 +88,7 @@ pkg_files( srcs = [ "BUILD.bazel", "Google.Protobuf.Conformance.csproj", - "Conformance.cs", + "Conformance.pb.cs", "Program.cs", ], strip_prefix = strip_prefix.from_root(""), diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs deleted file mode 100644 index 77fd9c9bbae2..000000000000 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ /dev/null @@ -1,1590 +0,0 @@ -// -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: conformance.proto -// -#pragma warning disable 1591, 0612, 3021, 8981 -#region Designer generated code - -using pb = global::Google.Protobuf; -using pbc = global::Google.Protobuf.Collections; -using pbr = global::Google.Protobuf.Reflection; -using scg = global::System.Collections.Generic; -namespace Conformance { - - /// Holder for reflection information generated from conformance.proto - public static partial class ConformanceReflection { - - #region Descriptor - /// File descriptor for conformance.proto - public static pbr::FileDescriptor Descriptor { - get { return descriptor; } - } - private static pbr::FileDescriptor descriptor; - - static ConformanceReflection() { - byte[] descriptorData = global::System.Convert.FromBase64String( - string.Concat( - "ChFjb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2UiHQoKRmFpbHVyZVNl", - "dBIPCgdmYWlsdXJlGAEgAygJIuMCChJDb25mb3JtYW5jZVJlcXVlc3QSGgoQ", - "cHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25fcGF5bG9hZBgCIAEo", - "CUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRleHRfcGF5bG9hZBgI", - "IAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0GAMgASgOMhcuY29u", - "Zm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5cGUYBCABKAkSMAoN", - "dGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNlLlRlc3RDYXRlZ29y", - "eRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsyHy5jb25mb3JtYW5j", - "ZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5rbm93bl9maWVsZHMY", - "CSABKAhCCQoHcGF5bG9hZCLhAQoTQ29uZm9ybWFuY2VSZXNwb25zZRIVCgtw", - "YXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9lcnJvchgGIAEoCUgA", - "EhcKDXJ1bnRpbWVfZXJyb3IYAiABKAlIABIaChBwcm90b2J1Zl9wYXlsb2Fk", - "GAMgASgMSAASFgoManNvbl9wYXlsb2FkGAQgASgJSAASEQoHc2tpcHBlZBgF", - "IAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRleHRfcGF5bG9h", - "ZBgIIAEoCUgAQggKBnJlc3VsdCI3ChJKc3BiRW5jb2RpbmdDb25maWcSIQoZ", - "dXNlX2pzcGJfYXJyYXlfYW55X2Zvcm1hdBgBIAEoCCpQCgpXaXJlRm9ybWF0", - "Eg8KC1VOU1BFQ0lGSUVEEAASDAoIUFJPVE9CVUYQARIICgRKU09OEAISCAoE", - "SlNQQhADEg8KC1RFWFRfRk9STUFUEAQqjwEKDFRlc3RDYXRlZ29yeRIUChBV", - "TlNQRUNJRklFRF9URVNUEAASDwoLQklOQVJZX1RFU1QQARINCglKU09OX1RF", - "U1QQAhIkCiBKU09OX0lHTk9SRV9VTktOT1dOX1BBUlNJTkdfVEVTVBADEg0K", - "CUpTUEJfVEVTVBAEEhQKEFRFWFRfRk9STUFUX1RFU1QQBUIhCh9jb20uZ29v", - "Z2xlLnByb3RvYnVmLmNvbmZvcm1hbmNlYgZwcm90bzM=")); - descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, - new pbr::FileDescriptor[] { }, - new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), typeof(global::Conformance.TestCategory), }, null, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Failure" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "JspbPayload", "TextPayload", "RequestedOutputFormat", "MessageType", "TestCategory", "JspbEncodingOptions", "PrintUnknownFields" }, new[]{ "Payload" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped", "JspbPayload", "TextPayload" }, new[]{ "Result" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.JspbEncodingConfig), global::Conformance.JspbEncodingConfig.Parser, new[]{ "UseJspbArrayAnyFormat" }, null, null, null, null) - })); - } - #endregion - - } - #region Enums - public enum WireFormat { - [pbr::OriginalName("UNSPECIFIED")] Unspecified = 0, - [pbr::OriginalName("PROTOBUF")] Protobuf = 1, - [pbr::OriginalName("JSON")] Json = 2, - /// - /// Only used inside Google. Opensource testees just skip it. - /// - [pbr::OriginalName("JSPB")] Jspb = 3, - [pbr::OriginalName("TEXT_FORMAT")] TextFormat = 4, - } - - public enum TestCategory { - [pbr::OriginalName("UNSPECIFIED_TEST")] UnspecifiedTest = 0, - /// - /// Test binary wire format. - /// - [pbr::OriginalName("BINARY_TEST")] BinaryTest = 1, - /// - /// Test json wire format. - /// - [pbr::OriginalName("JSON_TEST")] JsonTest = 2, - /// - /// Similar to JSON_TEST. However, during parsing json, testee should ignore - /// unknown fields. This feature is optional. Each implementation can decide - /// whether to support it. See - /// https://developers.google.com/protocol-buffers/docs/proto3#json_options - /// for more detail. - /// - [pbr::OriginalName("JSON_IGNORE_UNKNOWN_PARSING_TEST")] JsonIgnoreUnknownParsingTest = 3, - /// - /// Test jspb wire format. Only used inside Google. Opensource testees just skip it. - /// - [pbr::OriginalName("JSPB_TEST")] JspbTest = 4, - /// - /// Test text format. For cpp, java and python, testees can already deal with - /// this type. Testees of other languages can simply skip it. - /// - [pbr::OriginalName("TEXT_FORMAT_TEST")] TextFormatTest = 5, - } - - #endregion - - #region Messages - /// - /// The conformance runner will request a list of failures as the first request. - /// This will be known by message_type == "conformance.FailureSet", a conformance - /// test should return a serialized FailureSet in protobuf_payload. - /// - public sealed partial class FailureSet : pb::IMessage - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - , pb::IBufferMessage - #endif - { - private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new FailureSet()); - private pb::UnknownFieldSet _unknownFields; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pb::MessageParser Parser { get { return _parser; } } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - pbr::MessageDescriptor pb::IMessage.Descriptor { - get { return Descriptor; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public FailureSet() { - OnConstruction(); - } - - partial void OnConstruction(); - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public FailureSet(FailureSet other) : this() { - failure_ = other.failure_.Clone(); - _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public FailureSet Clone() { - return new FailureSet(this); - } - - /// Field number for the "failure" field. - public const int FailureFieldNumber = 1; - private static readonly pb::FieldCodec _repeated_failure_codec - = pb::FieldCodec.ForString(10); - private readonly pbc::RepeatedField failure_ = new pbc::RepeatedField(); - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public pbc::RepeatedField Failure { - get { return failure_; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override bool Equals(object other) { - return Equals(other as FailureSet); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool Equals(FailureSet other) { - if (ReferenceEquals(other, null)) { - return false; - } - if (ReferenceEquals(other, this)) { - return true; - } - if(!failure_.Equals(other.failure_)) return false; - return Equals(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override int GetHashCode() { - int hash = 1; - hash ^= failure_.GetHashCode(); - if (_unknownFields != null) { - hash ^= _unknownFields.GetHashCode(); - } - return hash; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override string ToString() { - return pb::JsonFormatter.ToDiagnosticString(this); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void WriteTo(pb::CodedOutputStream output) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - output.WriteRawMessage(this); - #else - failure_.WriteTo(output, _repeated_failure_codec); - if (_unknownFields != null) { - _unknownFields.WriteTo(output); - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - failure_.WriteTo(ref output, _repeated_failure_codec); - if (_unknownFields != null) { - _unknownFields.WriteTo(ref output); - } - } - #endif - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public int CalculateSize() { - int size = 0; - size += failure_.CalculateSize(_repeated_failure_codec); - if (_unknownFields != null) { - size += _unknownFields.CalculateSize(); - } - return size; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(FailureSet other) { - if (other == null) { - return; - } - failure_.Add(other.failure_); - _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(pb::CodedInputStream input) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - input.ReadRawMessage(this); - #else - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); - break; - case 10: { - failure_.AddEntriesFrom(input, _repeated_failure_codec); - break; - } - } - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); - break; - case 10: { - failure_.AddEntriesFrom(ref input, _repeated_failure_codec); - break; - } - } - } - } - #endif - - } - - /// - /// Represents a single test case's input. The testee should: - /// - /// 1. parse this proto (which should always succeed) - /// 2. parse the protobuf or JSON payload in "payload" (which may fail) - /// 3. if the parse succeeded, serialize the message in the requested format. - /// - public sealed partial class ConformanceRequest : pb::IMessage - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - , pb::IBufferMessage - #endif - { - private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new ConformanceRequest()); - private pb::UnknownFieldSet _unknownFields; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pb::MessageParser Parser { get { return _parser; } } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - pbr::MessageDescriptor pb::IMessage.Descriptor { - get { return Descriptor; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceRequest() { - OnConstruction(); - } - - partial void OnConstruction(); - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceRequest(ConformanceRequest other) : this() { - requestedOutputFormat_ = other.requestedOutputFormat_; - messageType_ = other.messageType_; - testCategory_ = other.testCategory_; - jspbEncodingOptions_ = other.jspbEncodingOptions_ != null ? other.jspbEncodingOptions_.Clone() : null; - printUnknownFields_ = other.printUnknownFields_; - switch (other.PayloadCase) { - case PayloadOneofCase.ProtobufPayload: - ProtobufPayload = other.ProtobufPayload; - break; - case PayloadOneofCase.JsonPayload: - JsonPayload = other.JsonPayload; - break; - case PayloadOneofCase.JspbPayload: - JspbPayload = other.JspbPayload; - break; - case PayloadOneofCase.TextPayload: - TextPayload = other.TextPayload; - break; - } - - _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceRequest Clone() { - return new ConformanceRequest(this); - } - - /// Field number for the "protobuf_payload" field. - public const int ProtobufPayloadFieldNumber = 1; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public pb::ByteString ProtobufPayload { - get { return payloadCase_ == PayloadOneofCase.ProtobufPayload ? (pb::ByteString) payload_ : pb::ByteString.Empty; } - set { - payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - payloadCase_ = PayloadOneofCase.ProtobufPayload; - } - } - - /// Field number for the "json_payload" field. - public const int JsonPayloadFieldNumber = 2; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string JsonPayload { - get { return payloadCase_ == PayloadOneofCase.JsonPayload ? (string) payload_ : ""; } - set { - payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - payloadCase_ = PayloadOneofCase.JsonPayload; - } - } - - /// Field number for the "jspb_payload" field. - public const int JspbPayloadFieldNumber = 7; - /// - /// Only used inside google. Opensource testees just skip it. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string JspbPayload { - get { return payloadCase_ == PayloadOneofCase.JspbPayload ? (string) payload_ : ""; } - set { - payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - payloadCase_ = PayloadOneofCase.JspbPayload; - } - } - - /// Field number for the "text_payload" field. - public const int TextPayloadFieldNumber = 8; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string TextPayload { - get { return payloadCase_ == PayloadOneofCase.TextPayload ? (string) payload_ : ""; } - set { - payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - payloadCase_ = PayloadOneofCase.TextPayload; - } - } - - /// Field number for the "requested_output_format" field. - public const int RequestedOutputFormatFieldNumber = 3; - private global::Conformance.WireFormat requestedOutputFormat_ = global::Conformance.WireFormat.Unspecified; - /// - /// Which format should the testee serialize its message to? - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public global::Conformance.WireFormat RequestedOutputFormat { - get { return requestedOutputFormat_; } - set { - requestedOutputFormat_ = value; - } - } - - /// Field number for the "message_type" field. - public const int MessageTypeFieldNumber = 4; - private string messageType_ = ""; - /// - /// The full name for the test message to use; for the moment, either: - /// protobuf_test_messages.proto3.TestAllTypesProto3 or - /// protobuf_test_messages.google.protobuf.TestAllTypesProto2. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string MessageType { - get { return messageType_; } - set { - messageType_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - } - } - - /// Field number for the "test_category" field. - public const int TestCategoryFieldNumber = 5; - private global::Conformance.TestCategory testCategory_ = global::Conformance.TestCategory.UnspecifiedTest; - /// - /// Each test is given a specific test category. Some category may need - /// specific support in testee programs. Refer to the definition of TestCategory - /// for more information. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public global::Conformance.TestCategory TestCategory { - get { return testCategory_; } - set { - testCategory_ = value; - } - } - - /// Field number for the "jspb_encoding_options" field. - public const int JspbEncodingOptionsFieldNumber = 6; - private global::Conformance.JspbEncodingConfig jspbEncodingOptions_; - /// - /// Specify details for how to encode jspb. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public global::Conformance.JspbEncodingConfig JspbEncodingOptions { - get { return jspbEncodingOptions_; } - set { - jspbEncodingOptions_ = value; - } - } - - /// Field number for the "print_unknown_fields" field. - public const int PrintUnknownFieldsFieldNumber = 9; - private bool printUnknownFields_; - /// - /// This can be used in json and text format. If true, testee should print - /// unknown fields instead of ignore. This feature is optional. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool PrintUnknownFields { - get { return printUnknownFields_; } - set { - printUnknownFields_ = value; - } - } - - private object payload_; - /// Enum of possible cases for the "payload" oneof. - public enum PayloadOneofCase { - None = 0, - ProtobufPayload = 1, - JsonPayload = 2, - JspbPayload = 7, - TextPayload = 8, - } - private PayloadOneofCase payloadCase_ = PayloadOneofCase.None; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public PayloadOneofCase PayloadCase { - get { return payloadCase_; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void ClearPayload() { - payloadCase_ = PayloadOneofCase.None; - payload_ = null; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override bool Equals(object other) { - return Equals(other as ConformanceRequest); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool Equals(ConformanceRequest other) { - if (ReferenceEquals(other, null)) { - return false; - } - if (ReferenceEquals(other, this)) { - return true; - } - if (ProtobufPayload != other.ProtobufPayload) return false; - if (JsonPayload != other.JsonPayload) return false; - if (JspbPayload != other.JspbPayload) return false; - if (TextPayload != other.TextPayload) return false; - if (RequestedOutputFormat != other.RequestedOutputFormat) return false; - if (MessageType != other.MessageType) return false; - if (TestCategory != other.TestCategory) return false; - if (!object.Equals(JspbEncodingOptions, other.JspbEncodingOptions)) return false; - if (PrintUnknownFields != other.PrintUnknownFields) return false; - if (PayloadCase != other.PayloadCase) return false; - return Equals(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override int GetHashCode() { - int hash = 1; - if (payloadCase_ == PayloadOneofCase.ProtobufPayload) hash ^= ProtobufPayload.GetHashCode(); - if (payloadCase_ == PayloadOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); - if (payloadCase_ == PayloadOneofCase.JspbPayload) hash ^= JspbPayload.GetHashCode(); - if (payloadCase_ == PayloadOneofCase.TextPayload) hash ^= TextPayload.GetHashCode(); - if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) hash ^= RequestedOutputFormat.GetHashCode(); - if (MessageType.Length != 0) hash ^= MessageType.GetHashCode(); - if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) hash ^= TestCategory.GetHashCode(); - if (jspbEncodingOptions_ != null) hash ^= JspbEncodingOptions.GetHashCode(); - if (PrintUnknownFields != false) hash ^= PrintUnknownFields.GetHashCode(); - hash ^= (int) payloadCase_; - if (_unknownFields != null) { - hash ^= _unknownFields.GetHashCode(); - } - return hash; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override string ToString() { - return pb::JsonFormatter.ToDiagnosticString(this); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void WriteTo(pb::CodedOutputStream output) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - output.WriteRawMessage(this); - #else - if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { - output.WriteRawTag(10); - output.WriteBytes(ProtobufPayload); - } - if (payloadCase_ == PayloadOneofCase.JsonPayload) { - output.WriteRawTag(18); - output.WriteString(JsonPayload); - } - if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { - output.WriteRawTag(24); - output.WriteEnum((int) RequestedOutputFormat); - } - if (MessageType.Length != 0) { - output.WriteRawTag(34); - output.WriteString(MessageType); - } - if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { - output.WriteRawTag(40); - output.WriteEnum((int) TestCategory); - } - if (jspbEncodingOptions_ != null) { - output.WriteRawTag(50); - output.WriteMessage(JspbEncodingOptions); - } - if (payloadCase_ == PayloadOneofCase.JspbPayload) { - output.WriteRawTag(58); - output.WriteString(JspbPayload); - } - if (payloadCase_ == PayloadOneofCase.TextPayload) { - output.WriteRawTag(66); - output.WriteString(TextPayload); - } - if (PrintUnknownFields != false) { - output.WriteRawTag(72); - output.WriteBool(PrintUnknownFields); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(output); - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { - output.WriteRawTag(10); - output.WriteBytes(ProtobufPayload); - } - if (payloadCase_ == PayloadOneofCase.JsonPayload) { - output.WriteRawTag(18); - output.WriteString(JsonPayload); - } - if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { - output.WriteRawTag(24); - output.WriteEnum((int) RequestedOutputFormat); - } - if (MessageType.Length != 0) { - output.WriteRawTag(34); - output.WriteString(MessageType); - } - if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { - output.WriteRawTag(40); - output.WriteEnum((int) TestCategory); - } - if (jspbEncodingOptions_ != null) { - output.WriteRawTag(50); - output.WriteMessage(JspbEncodingOptions); - } - if (payloadCase_ == PayloadOneofCase.JspbPayload) { - output.WriteRawTag(58); - output.WriteString(JspbPayload); - } - if (payloadCase_ == PayloadOneofCase.TextPayload) { - output.WriteRawTag(66); - output.WriteString(TextPayload); - } - if (PrintUnknownFields != false) { - output.WriteRawTag(72); - output.WriteBool(PrintUnknownFields); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(ref output); - } - } - #endif - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public int CalculateSize() { - int size = 0; - if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { - size += 1 + pb::CodedOutputStream.ComputeBytesSize(ProtobufPayload); - } - if (payloadCase_ == PayloadOneofCase.JsonPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(JsonPayload); - } - if (payloadCase_ == PayloadOneofCase.JspbPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(JspbPayload); - } - if (payloadCase_ == PayloadOneofCase.TextPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(TextPayload); - } - if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) RequestedOutputFormat); - } - if (MessageType.Length != 0) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(MessageType); - } - if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) TestCategory); - } - if (jspbEncodingOptions_ != null) { - size += 1 + pb::CodedOutputStream.ComputeMessageSize(JspbEncodingOptions); - } - if (PrintUnknownFields != false) { - size += 1 + 1; - } - if (_unknownFields != null) { - size += _unknownFields.CalculateSize(); - } - return size; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(ConformanceRequest other) { - if (other == null) { - return; - } - if (other.RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { - RequestedOutputFormat = other.RequestedOutputFormat; - } - if (other.MessageType.Length != 0) { - MessageType = other.MessageType; - } - if (other.TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { - TestCategory = other.TestCategory; - } - if (other.jspbEncodingOptions_ != null) { - if (jspbEncodingOptions_ == null) { - JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); - } - JspbEncodingOptions.MergeFrom(other.JspbEncodingOptions); - } - if (other.PrintUnknownFields != false) { - PrintUnknownFields = other.PrintUnknownFields; - } - switch (other.PayloadCase) { - case PayloadOneofCase.ProtobufPayload: - ProtobufPayload = other.ProtobufPayload; - break; - case PayloadOneofCase.JsonPayload: - JsonPayload = other.JsonPayload; - break; - case PayloadOneofCase.JspbPayload: - JspbPayload = other.JspbPayload; - break; - case PayloadOneofCase.TextPayload: - TextPayload = other.TextPayload; - break; - } - - _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(pb::CodedInputStream input) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - input.ReadRawMessage(this); - #else - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); - break; - case 10: { - ProtobufPayload = input.ReadBytes(); - break; - } - case 18: { - JsonPayload = input.ReadString(); - break; - } - case 24: { - RequestedOutputFormat = (global::Conformance.WireFormat) input.ReadEnum(); - break; - } - case 34: { - MessageType = input.ReadString(); - break; - } - case 40: { - TestCategory = (global::Conformance.TestCategory) input.ReadEnum(); - break; - } - case 50: { - if (jspbEncodingOptions_ == null) { - JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); - } - input.ReadMessage(JspbEncodingOptions); - break; - } - case 58: { - JspbPayload = input.ReadString(); - break; - } - case 66: { - TextPayload = input.ReadString(); - break; - } - case 72: { - PrintUnknownFields = input.ReadBool(); - break; - } - } - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); - break; - case 10: { - ProtobufPayload = input.ReadBytes(); - break; - } - case 18: { - JsonPayload = input.ReadString(); - break; - } - case 24: { - RequestedOutputFormat = (global::Conformance.WireFormat) input.ReadEnum(); - break; - } - case 34: { - MessageType = input.ReadString(); - break; - } - case 40: { - TestCategory = (global::Conformance.TestCategory) input.ReadEnum(); - break; - } - case 50: { - if (jspbEncodingOptions_ == null) { - JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); - } - input.ReadMessage(JspbEncodingOptions); - break; - } - case 58: { - JspbPayload = input.ReadString(); - break; - } - case 66: { - TextPayload = input.ReadString(); - break; - } - case 72: { - PrintUnknownFields = input.ReadBool(); - break; - } - } - } - } - #endif - - } - - /// - /// Represents a single test case's output. - /// - public sealed partial class ConformanceResponse : pb::IMessage - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - , pb::IBufferMessage - #endif - { - private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new ConformanceResponse()); - private pb::UnknownFieldSet _unknownFields; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pb::MessageParser Parser { get { return _parser; } } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - pbr::MessageDescriptor pb::IMessage.Descriptor { - get { return Descriptor; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceResponse() { - OnConstruction(); - } - - partial void OnConstruction(); - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceResponse(ConformanceResponse other) : this() { - switch (other.ResultCase) { - case ResultOneofCase.ParseError: - ParseError = other.ParseError; - break; - case ResultOneofCase.SerializeError: - SerializeError = other.SerializeError; - break; - case ResultOneofCase.RuntimeError: - RuntimeError = other.RuntimeError; - break; - case ResultOneofCase.ProtobufPayload: - ProtobufPayload = other.ProtobufPayload; - break; - case ResultOneofCase.JsonPayload: - JsonPayload = other.JsonPayload; - break; - case ResultOneofCase.Skipped: - Skipped = other.Skipped; - break; - case ResultOneofCase.JspbPayload: - JspbPayload = other.JspbPayload; - break; - case ResultOneofCase.TextPayload: - TextPayload = other.TextPayload; - break; - } - - _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ConformanceResponse Clone() { - return new ConformanceResponse(this); - } - - /// Field number for the "parse_error" field. - public const int ParseErrorFieldNumber = 1; - /// - /// This string should be set to indicate parsing failed. The string can - /// provide more information about the parse error if it is available. - /// - /// Setting this string does not necessarily mean the testee failed the - /// test. Some of the test cases are intentionally invalid input. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string ParseError { - get { return resultCase_ == ResultOneofCase.ParseError ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.ParseError; - } - } - - /// Field number for the "serialize_error" field. - public const int SerializeErrorFieldNumber = 6; - /// - /// If the input was successfully parsed but errors occurred when - /// serializing it to the requested output format, set the error message in - /// this field. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string SerializeError { - get { return resultCase_ == ResultOneofCase.SerializeError ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.SerializeError; - } - } - - /// Field number for the "runtime_error" field. - public const int RuntimeErrorFieldNumber = 2; - /// - /// This should be set if some other error occurred. This will always - /// indicate that the test failed. The string can provide more information - /// about the failure. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string RuntimeError { - get { return resultCase_ == ResultOneofCase.RuntimeError ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.RuntimeError; - } - } - - /// Field number for the "protobuf_payload" field. - public const int ProtobufPayloadFieldNumber = 3; - /// - /// If the input was successfully parsed and the requested output was - /// protobuf, serialize it to protobuf and set it in this field. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public pb::ByteString ProtobufPayload { - get { return resultCase_ == ResultOneofCase.ProtobufPayload ? (pb::ByteString) result_ : pb::ByteString.Empty; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.ProtobufPayload; - } - } - - /// Field number for the "json_payload" field. - public const int JsonPayloadFieldNumber = 4; - /// - /// If the input was successfully parsed and the requested output was JSON, - /// serialize to JSON and set it in this field. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string JsonPayload { - get { return resultCase_ == ResultOneofCase.JsonPayload ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.JsonPayload; - } - } - - /// Field number for the "skipped" field. - public const int SkippedFieldNumber = 5; - /// - /// For when the testee skipped the test, likely because a certain feature - /// wasn't supported, like JSON input/output. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string Skipped { - get { return resultCase_ == ResultOneofCase.Skipped ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.Skipped; - } - } - - /// Field number for the "jspb_payload" field. - public const int JspbPayloadFieldNumber = 7; - /// - /// If the input was successfully parsed and the requested output was JSPB, - /// serialize to JSPB and set it in this field. JSPB is only used inside - /// Google. Opensource testees can just skip it. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string JspbPayload { - get { return resultCase_ == ResultOneofCase.JspbPayload ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.JspbPayload; - } - } - - /// Field number for the "text_payload" field. - public const int TextPayloadFieldNumber = 8; - /// - /// If the input was successfully parsed and the requested output was - /// TEXT_FORMAT, serialize to TEXT_FORMAT and set it in this field. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string TextPayload { - get { return resultCase_ == ResultOneofCase.TextPayload ? (string) result_ : ""; } - set { - result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); - resultCase_ = ResultOneofCase.TextPayload; - } - } - - private object result_; - /// Enum of possible cases for the "result" oneof. - public enum ResultOneofCase { - None = 0, - ParseError = 1, - SerializeError = 6, - RuntimeError = 2, - ProtobufPayload = 3, - JsonPayload = 4, - Skipped = 5, - JspbPayload = 7, - TextPayload = 8, - } - private ResultOneofCase resultCase_ = ResultOneofCase.None; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public ResultOneofCase ResultCase { - get { return resultCase_; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void ClearResult() { - resultCase_ = ResultOneofCase.None; - result_ = null; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override bool Equals(object other) { - return Equals(other as ConformanceResponse); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool Equals(ConformanceResponse other) { - if (ReferenceEquals(other, null)) { - return false; - } - if (ReferenceEquals(other, this)) { - return true; - } - if (ParseError != other.ParseError) return false; - if (SerializeError != other.SerializeError) return false; - if (RuntimeError != other.RuntimeError) return false; - if (ProtobufPayload != other.ProtobufPayload) return false; - if (JsonPayload != other.JsonPayload) return false; - if (Skipped != other.Skipped) return false; - if (JspbPayload != other.JspbPayload) return false; - if (TextPayload != other.TextPayload) return false; - if (ResultCase != other.ResultCase) return false; - return Equals(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override int GetHashCode() { - int hash = 1; - if (resultCase_ == ResultOneofCase.ParseError) hash ^= ParseError.GetHashCode(); - if (resultCase_ == ResultOneofCase.SerializeError) hash ^= SerializeError.GetHashCode(); - if (resultCase_ == ResultOneofCase.RuntimeError) hash ^= RuntimeError.GetHashCode(); - if (resultCase_ == ResultOneofCase.ProtobufPayload) hash ^= ProtobufPayload.GetHashCode(); - if (resultCase_ == ResultOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); - if (resultCase_ == ResultOneofCase.Skipped) hash ^= Skipped.GetHashCode(); - if (resultCase_ == ResultOneofCase.JspbPayload) hash ^= JspbPayload.GetHashCode(); - if (resultCase_ == ResultOneofCase.TextPayload) hash ^= TextPayload.GetHashCode(); - hash ^= (int) resultCase_; - if (_unknownFields != null) { - hash ^= _unknownFields.GetHashCode(); - } - return hash; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override string ToString() { - return pb::JsonFormatter.ToDiagnosticString(this); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void WriteTo(pb::CodedOutputStream output) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - output.WriteRawMessage(this); - #else - if (resultCase_ == ResultOneofCase.ParseError) { - output.WriteRawTag(10); - output.WriteString(ParseError); - } - if (resultCase_ == ResultOneofCase.RuntimeError) { - output.WriteRawTag(18); - output.WriteString(RuntimeError); - } - if (resultCase_ == ResultOneofCase.ProtobufPayload) { - output.WriteRawTag(26); - output.WriteBytes(ProtobufPayload); - } - if (resultCase_ == ResultOneofCase.JsonPayload) { - output.WriteRawTag(34); - output.WriteString(JsonPayload); - } - if (resultCase_ == ResultOneofCase.Skipped) { - output.WriteRawTag(42); - output.WriteString(Skipped); - } - if (resultCase_ == ResultOneofCase.SerializeError) { - output.WriteRawTag(50); - output.WriteString(SerializeError); - } - if (resultCase_ == ResultOneofCase.JspbPayload) { - output.WriteRawTag(58); - output.WriteString(JspbPayload); - } - if (resultCase_ == ResultOneofCase.TextPayload) { - output.WriteRawTag(66); - output.WriteString(TextPayload); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(output); - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - if (resultCase_ == ResultOneofCase.ParseError) { - output.WriteRawTag(10); - output.WriteString(ParseError); - } - if (resultCase_ == ResultOneofCase.RuntimeError) { - output.WriteRawTag(18); - output.WriteString(RuntimeError); - } - if (resultCase_ == ResultOneofCase.ProtobufPayload) { - output.WriteRawTag(26); - output.WriteBytes(ProtobufPayload); - } - if (resultCase_ == ResultOneofCase.JsonPayload) { - output.WriteRawTag(34); - output.WriteString(JsonPayload); - } - if (resultCase_ == ResultOneofCase.Skipped) { - output.WriteRawTag(42); - output.WriteString(Skipped); - } - if (resultCase_ == ResultOneofCase.SerializeError) { - output.WriteRawTag(50); - output.WriteString(SerializeError); - } - if (resultCase_ == ResultOneofCase.JspbPayload) { - output.WriteRawTag(58); - output.WriteString(JspbPayload); - } - if (resultCase_ == ResultOneofCase.TextPayload) { - output.WriteRawTag(66); - output.WriteString(TextPayload); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(ref output); - } - } - #endif - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public int CalculateSize() { - int size = 0; - if (resultCase_ == ResultOneofCase.ParseError) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(ParseError); - } - if (resultCase_ == ResultOneofCase.SerializeError) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(SerializeError); - } - if (resultCase_ == ResultOneofCase.RuntimeError) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(RuntimeError); - } - if (resultCase_ == ResultOneofCase.ProtobufPayload) { - size += 1 + pb::CodedOutputStream.ComputeBytesSize(ProtobufPayload); - } - if (resultCase_ == ResultOneofCase.JsonPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(JsonPayload); - } - if (resultCase_ == ResultOneofCase.Skipped) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(Skipped); - } - if (resultCase_ == ResultOneofCase.JspbPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(JspbPayload); - } - if (resultCase_ == ResultOneofCase.TextPayload) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(TextPayload); - } - if (_unknownFields != null) { - size += _unknownFields.CalculateSize(); - } - return size; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(ConformanceResponse other) { - if (other == null) { - return; - } - switch (other.ResultCase) { - case ResultOneofCase.ParseError: - ParseError = other.ParseError; - break; - case ResultOneofCase.SerializeError: - SerializeError = other.SerializeError; - break; - case ResultOneofCase.RuntimeError: - RuntimeError = other.RuntimeError; - break; - case ResultOneofCase.ProtobufPayload: - ProtobufPayload = other.ProtobufPayload; - break; - case ResultOneofCase.JsonPayload: - JsonPayload = other.JsonPayload; - break; - case ResultOneofCase.Skipped: - Skipped = other.Skipped; - break; - case ResultOneofCase.JspbPayload: - JspbPayload = other.JspbPayload; - break; - case ResultOneofCase.TextPayload: - TextPayload = other.TextPayload; - break; - } - - _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(pb::CodedInputStream input) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - input.ReadRawMessage(this); - #else - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); - break; - case 10: { - ParseError = input.ReadString(); - break; - } - case 18: { - RuntimeError = input.ReadString(); - break; - } - case 26: { - ProtobufPayload = input.ReadBytes(); - break; - } - case 34: { - JsonPayload = input.ReadString(); - break; - } - case 42: { - Skipped = input.ReadString(); - break; - } - case 50: { - SerializeError = input.ReadString(); - break; - } - case 58: { - JspbPayload = input.ReadString(); - break; - } - case 66: { - TextPayload = input.ReadString(); - break; - } - } - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); - break; - case 10: { - ParseError = input.ReadString(); - break; - } - case 18: { - RuntimeError = input.ReadString(); - break; - } - case 26: { - ProtobufPayload = input.ReadBytes(); - break; - } - case 34: { - JsonPayload = input.ReadString(); - break; - } - case 42: { - Skipped = input.ReadString(); - break; - } - case 50: { - SerializeError = input.ReadString(); - break; - } - case 58: { - JspbPayload = input.ReadString(); - break; - } - case 66: { - TextPayload = input.ReadString(); - break; - } - } - } - } - #endif - - } - - /// - /// Encoding options for jspb format. - /// - public sealed partial class JspbEncodingConfig : pb::IMessage - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - , pb::IBufferMessage - #endif - { - private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new JspbEncodingConfig()); - private pb::UnknownFieldSet _unknownFields; - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pb::MessageParser Parser { get { return _parser; } } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - pbr::MessageDescriptor pb::IMessage.Descriptor { - get { return Descriptor; } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public JspbEncodingConfig() { - OnConstruction(); - } - - partial void OnConstruction(); - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public JspbEncodingConfig(JspbEncodingConfig other) : this() { - useJspbArrayAnyFormat_ = other.useJspbArrayAnyFormat_; - _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public JspbEncodingConfig Clone() { - return new JspbEncodingConfig(this); - } - - /// Field number for the "use_jspb_array_any_format" field. - public const int UseJspbArrayAnyFormatFieldNumber = 1; - private bool useJspbArrayAnyFormat_; - /// - /// Encode the value field of Any as jspb array if true, otherwise binary. - /// - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool UseJspbArrayAnyFormat { - get { return useJspbArrayAnyFormat_; } - set { - useJspbArrayAnyFormat_ = value; - } - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override bool Equals(object other) { - return Equals(other as JspbEncodingConfig); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public bool Equals(JspbEncodingConfig other) { - if (ReferenceEquals(other, null)) { - return false; - } - if (ReferenceEquals(other, this)) { - return true; - } - if (UseJspbArrayAnyFormat != other.UseJspbArrayAnyFormat) return false; - return Equals(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override int GetHashCode() { - int hash = 1; - if (UseJspbArrayAnyFormat != false) hash ^= UseJspbArrayAnyFormat.GetHashCode(); - if (_unknownFields != null) { - hash ^= _unknownFields.GetHashCode(); - } - return hash; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public override string ToString() { - return pb::JsonFormatter.ToDiagnosticString(this); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void WriteTo(pb::CodedOutputStream output) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - output.WriteRawMessage(this); - #else - if (UseJspbArrayAnyFormat != false) { - output.WriteRawTag(8); - output.WriteBool(UseJspbArrayAnyFormat); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(output); - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - if (UseJspbArrayAnyFormat != false) { - output.WriteRawTag(8); - output.WriteBool(UseJspbArrayAnyFormat); - } - if (_unknownFields != null) { - _unknownFields.WriteTo(ref output); - } - } - #endif - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public int CalculateSize() { - int size = 0; - if (UseJspbArrayAnyFormat != false) { - size += 1 + 1; - } - if (_unknownFields != null) { - size += _unknownFields.CalculateSize(); - } - return size; - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(JspbEncodingConfig other) { - if (other == null) { - return; - } - if (other.UseJspbArrayAnyFormat != false) { - UseJspbArrayAnyFormat = other.UseJspbArrayAnyFormat; - } - _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); - } - - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public void MergeFrom(pb::CodedInputStream input) { - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - input.ReadRawMessage(this); - #else - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); - break; - case 8: { - UseJspbArrayAnyFormat = input.ReadBool(); - break; - } - } - } - #endif - } - - #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { - uint tag; - while ((tag = input.ReadTag()) != 0) { - switch(tag) { - default: - _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); - break; - case 8: { - UseJspbArrayAnyFormat = input.ReadBool(); - break; - } - } - } - } - #endif - - } - - #endregion - -} - -#endregion Designer generated code From a20a537f1e003c0426a2f31b9491b81d3b5f2496 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 14 Feb 2023 09:05:33 -0800 Subject: [PATCH 033/463] Fixing broken rust package PiperOrigin-RevId: 509546035 --- rust/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/BUILD.bazel b/rust/BUILD.bazel index c8dbca76ec24..5efdab5493f7 100644 --- a/rust/BUILD.bazel +++ b/rust/BUILD.bazel @@ -1,8 +1,8 @@ # Protobuf Rust runtime packages. -load("@rules_rust//:defs.bzl", "rust_library") +load("@rules_rust//rust:defs.bzl", "rust_library") -package(default_visibility = "//src/google/protobuf:__subpackages__") +package(default_visibility = ["//src/google/protobuf:__subpackages__"]) rust_library( name = "protobuf", From 3219a0b0d9bdacf67426a6c5afc95875fa7a38fe Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 Feb 2023 17:19:38 +0000 Subject: [PATCH 034/463] Auto-generate files after cl/509546035 --- .../Conformance.pb.cs | 1637 +++++++++++++++++ 1 file changed, 1637 insertions(+) create mode 100644 csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs new file mode 100644 index 000000000000..664a17b63684 --- /dev/null +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs @@ -0,0 +1,1637 @@ +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: conformance/conformance.proto +// +#pragma warning disable 1591, 0612, 3021, 8981 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace Conformance { + + /// Holder for reflection information generated from conformance/conformance.proto + public static partial class ConformanceReflection { + + #region Descriptor + /// File descriptor for conformance/conformance.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static ConformanceReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "Ch1jb25mb3JtYW5jZS9jb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2Ui", + "HQoKRmFpbHVyZVNldBIPCgdmYWlsdXJlGAEgAygJIuMCChJDb25mb3JtYW5j", + "ZVJlcXVlc3QSGgoQcHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25f", + "cGF5bG9hZBgCIAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRl", + "eHRfcGF5bG9hZBgIIAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0", + "GAMgASgOMhcuY29uZm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5", + "cGUYBCABKAkSMAoNdGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNl", + "LlRlc3RDYXRlZ29yeRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsy", + "Hy5jb25mb3JtYW5jZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5r", + "bm93bl9maWVsZHMYCSABKAhCCQoHcGF5bG9hZCL6AQoTQ29uZm9ybWFuY2VS", + "ZXNwb25zZRIVCgtwYXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9l", + "cnJvchgGIAEoCUgAEhcKDXRpbWVvdXRfZXJyb3IYCSABKAlIABIXCg1ydW50", + "aW1lX2Vycm9yGAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgA", + "EhYKDGpzb25fcGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIABIW", + "Cgxqc3BiX3BheWxvYWQYByABKAlIABIWCgx0ZXh0X3BheWxvYWQYCCABKAlI", + "AEIICgZyZXN1bHQiNwoSSnNwYkVuY29kaW5nQ29uZmlnEiEKGXVzZV9qc3Bi", + "X2FycmF5X2FueV9mb3JtYXQYASABKAgqUAoKV2lyZUZvcm1hdBIPCgtVTlNQ", + "RUNJRklFRBAAEgwKCFBST1RPQlVGEAESCAoESlNPThACEggKBEpTUEIQAxIP", + "CgtURVhUX0ZPUk1BVBAEKo8BCgxUZXN0Q2F0ZWdvcnkSFAoQVU5TUEVDSUZJ", + "RURfVEVTVBAAEg8KC0JJTkFSWV9URVNUEAESDQoJSlNPTl9URVNUEAISJAog", + "SlNPTl9JR05PUkVfVU5LTk9XTl9QQVJTSU5HX1RFU1QQAxINCglKU1BCX1RF", + "U1QQBBIUChBURVhUX0ZPUk1BVF9URVNUEAVCLwofY29tLmdvb2dsZS5wcm90", + "b2J1Zi5jb25mb3JtYW5jZaICC0NvbmZvcm1hbmNlYgZwcm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { }, + new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), typeof(global::Conformance.TestCategory), }, null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Failure" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "JspbPayload", "TextPayload", "RequestedOutputFormat", "MessageType", "TestCategory", "JspbEncodingOptions", "PrintUnknownFields" }, new[]{ "Payload" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "TimeoutError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped", "JspbPayload", "TextPayload" }, new[]{ "Result" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.JspbEncodingConfig), global::Conformance.JspbEncodingConfig.Parser, new[]{ "UseJspbArrayAnyFormat" }, null, null, null, null) + })); + } + #endregion + + } + #region Enums + public enum WireFormat { + [pbr::OriginalName("UNSPECIFIED")] Unspecified = 0, + [pbr::OriginalName("PROTOBUF")] Protobuf = 1, + [pbr::OriginalName("JSON")] Json = 2, + /// + /// Only used inside Google. Opensource testees just skip it. + /// + [pbr::OriginalName("JSPB")] Jspb = 3, + [pbr::OriginalName("TEXT_FORMAT")] TextFormat = 4, + } + + public enum TestCategory { + [pbr::OriginalName("UNSPECIFIED_TEST")] UnspecifiedTest = 0, + /// + /// Test binary wire format. + /// + [pbr::OriginalName("BINARY_TEST")] BinaryTest = 1, + /// + /// Test json wire format. + /// + [pbr::OriginalName("JSON_TEST")] JsonTest = 2, + /// + /// Similar to JSON_TEST. However, during parsing json, testee should ignore + /// unknown fields. This feature is optional. Each implementation can decide + /// whether to support it. See + /// https://developers.google.com/protocol-buffers/docs/proto3#json_options + /// for more detail. + /// + [pbr::OriginalName("JSON_IGNORE_UNKNOWN_PARSING_TEST")] JsonIgnoreUnknownParsingTest = 3, + /// + /// Test jspb wire format. Only used inside Google. Opensource testees just + /// skip it. + /// + [pbr::OriginalName("JSPB_TEST")] JspbTest = 4, + /// + /// Test text format. For cpp, java and python, testees can already deal with + /// this type. Testees of other languages can simply skip it. + /// + [pbr::OriginalName("TEXT_FORMAT_TEST")] TextFormatTest = 5, + } + + #endregion + + #region Messages + /// + /// The conformance runner will request a list of failures as the first request. + /// This will be known by message_type == "conformance.FailureSet", a conformance + /// test should return a serialized FailureSet in protobuf_payload. + /// + public sealed partial class FailureSet : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new FailureSet()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public FailureSet() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public FailureSet(FailureSet other) : this() { + failure_ = other.failure_.Clone(); + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public FailureSet Clone() { + return new FailureSet(this); + } + + /// Field number for the "failure" field. + public const int FailureFieldNumber = 1; + private static readonly pb::FieldCodec _repeated_failure_codec + = pb::FieldCodec.ForString(10); + private readonly pbc::RepeatedField failure_ = new pbc::RepeatedField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pbc::RepeatedField Failure { + get { return failure_; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as FailureSet); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(FailureSet other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if(!failure_.Equals(other.failure_)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + hash ^= failure_.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + failure_.WriteTo(output, _repeated_failure_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + failure_.WriteTo(ref output, _repeated_failure_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + size += failure_.CalculateSize(_repeated_failure_codec); + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(FailureSet other) { + if (other == null) { + return; + } + failure_.Add(other.failure_); + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + failure_.AddEntriesFrom(input, _repeated_failure_codec); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + failure_.AddEntriesFrom(ref input, _repeated_failure_codec); + break; + } + } + } + } + #endif + + } + + /// + /// Represents a single test case's input. The testee should: + /// + /// 1. parse this proto (which should always succeed) + /// 2. parse the protobuf or JSON payload in "payload" (which may fail) + /// 3. if the parse succeeded, serialize the message in the requested format. + /// + public sealed partial class ConformanceRequest : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new ConformanceRequest()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceRequest() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceRequest(ConformanceRequest other) : this() { + requestedOutputFormat_ = other.requestedOutputFormat_; + messageType_ = other.messageType_; + testCategory_ = other.testCategory_; + jspbEncodingOptions_ = other.jspbEncodingOptions_ != null ? other.jspbEncodingOptions_.Clone() : null; + printUnknownFields_ = other.printUnknownFields_; + switch (other.PayloadCase) { + case PayloadOneofCase.ProtobufPayload: + ProtobufPayload = other.ProtobufPayload; + break; + case PayloadOneofCase.JsonPayload: + JsonPayload = other.JsonPayload; + break; + case PayloadOneofCase.JspbPayload: + JspbPayload = other.JspbPayload; + break; + case PayloadOneofCase.TextPayload: + TextPayload = other.TextPayload; + break; + } + + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceRequest Clone() { + return new ConformanceRequest(this); + } + + /// Field number for the "protobuf_payload" field. + public const int ProtobufPayloadFieldNumber = 1; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString ProtobufPayload { + get { return payloadCase_ == PayloadOneofCase.ProtobufPayload ? (pb::ByteString) payload_ : pb::ByteString.Empty; } + set { + payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + payloadCase_ = PayloadOneofCase.ProtobufPayload; + } + } + + /// Field number for the "json_payload" field. + public const int JsonPayloadFieldNumber = 2; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string JsonPayload { + get { return payloadCase_ == PayloadOneofCase.JsonPayload ? (string) payload_ : ""; } + set { + payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + payloadCase_ = PayloadOneofCase.JsonPayload; + } + } + + /// Field number for the "jspb_payload" field. + public const int JspbPayloadFieldNumber = 7; + /// + /// Only used inside Google. Opensource testees just skip it. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string JspbPayload { + get { return payloadCase_ == PayloadOneofCase.JspbPayload ? (string) payload_ : ""; } + set { + payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + payloadCase_ = PayloadOneofCase.JspbPayload; + } + } + + /// Field number for the "text_payload" field. + public const int TextPayloadFieldNumber = 8; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string TextPayload { + get { return payloadCase_ == PayloadOneofCase.TextPayload ? (string) payload_ : ""; } + set { + payload_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + payloadCase_ = PayloadOneofCase.TextPayload; + } + } + + /// Field number for the "requested_output_format" field. + public const int RequestedOutputFormatFieldNumber = 3; + private global::Conformance.WireFormat requestedOutputFormat_ = global::Conformance.WireFormat.Unspecified; + /// + /// Which format should the testee serialize its message to? + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Conformance.WireFormat RequestedOutputFormat { + get { return requestedOutputFormat_; } + set { + requestedOutputFormat_ = value; + } + } + + /// Field number for the "message_type" field. + public const int MessageTypeFieldNumber = 4; + private string messageType_ = ""; + /// + /// The full name for the test message to use; for the moment, either: + /// protobuf_test_messages.proto3.TestAllTypesProto3 or + /// protobuf_test_messages.google.protobuf.TestAllTypesProto2. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string MessageType { + get { return messageType_; } + set { + messageType_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "test_category" field. + public const int TestCategoryFieldNumber = 5; + private global::Conformance.TestCategory testCategory_ = global::Conformance.TestCategory.UnspecifiedTest; + /// + /// Each test is given a specific test category. Some category may need + /// specific support in testee programs. Refer to the definition of + /// TestCategory for more information. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Conformance.TestCategory TestCategory { + get { return testCategory_; } + set { + testCategory_ = value; + } + } + + /// Field number for the "jspb_encoding_options" field. + public const int JspbEncodingOptionsFieldNumber = 6; + private global::Conformance.JspbEncodingConfig jspbEncodingOptions_; + /// + /// Specify details for how to encode jspb. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Conformance.JspbEncodingConfig JspbEncodingOptions { + get { return jspbEncodingOptions_; } + set { + jspbEncodingOptions_ = value; + } + } + + /// Field number for the "print_unknown_fields" field. + public const int PrintUnknownFieldsFieldNumber = 9; + private bool printUnknownFields_; + /// + /// This can be used in json and text format. If true, testee should print + /// unknown fields instead of ignore. This feature is optional. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool PrintUnknownFields { + get { return printUnknownFields_; } + set { + printUnknownFields_ = value; + } + } + + private object payload_; + /// Enum of possible cases for the "payload" oneof. + public enum PayloadOneofCase { + None = 0, + ProtobufPayload = 1, + JsonPayload = 2, + JspbPayload = 7, + TextPayload = 8, + } + private PayloadOneofCase payloadCase_ = PayloadOneofCase.None; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public PayloadOneofCase PayloadCase { + get { return payloadCase_; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearPayload() { + payloadCase_ = PayloadOneofCase.None; + payload_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as ConformanceRequest); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(ConformanceRequest other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (ProtobufPayload != other.ProtobufPayload) return false; + if (JsonPayload != other.JsonPayload) return false; + if (JspbPayload != other.JspbPayload) return false; + if (TextPayload != other.TextPayload) return false; + if (RequestedOutputFormat != other.RequestedOutputFormat) return false; + if (MessageType != other.MessageType) return false; + if (TestCategory != other.TestCategory) return false; + if (!object.Equals(JspbEncodingOptions, other.JspbEncodingOptions)) return false; + if (PrintUnknownFields != other.PrintUnknownFields) return false; + if (PayloadCase != other.PayloadCase) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (payloadCase_ == PayloadOneofCase.ProtobufPayload) hash ^= ProtobufPayload.GetHashCode(); + if (payloadCase_ == PayloadOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); + if (payloadCase_ == PayloadOneofCase.JspbPayload) hash ^= JspbPayload.GetHashCode(); + if (payloadCase_ == PayloadOneofCase.TextPayload) hash ^= TextPayload.GetHashCode(); + if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) hash ^= RequestedOutputFormat.GetHashCode(); + if (MessageType.Length != 0) hash ^= MessageType.GetHashCode(); + if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) hash ^= TestCategory.GetHashCode(); + if (jspbEncodingOptions_ != null) hash ^= JspbEncodingOptions.GetHashCode(); + if (PrintUnknownFields != false) hash ^= PrintUnknownFields.GetHashCode(); + hash ^= (int) payloadCase_; + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { + output.WriteRawTag(10); + output.WriteBytes(ProtobufPayload); + } + if (payloadCase_ == PayloadOneofCase.JsonPayload) { + output.WriteRawTag(18); + output.WriteString(JsonPayload); + } + if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { + output.WriteRawTag(24); + output.WriteEnum((int) RequestedOutputFormat); + } + if (MessageType.Length != 0) { + output.WriteRawTag(34); + output.WriteString(MessageType); + } + if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { + output.WriteRawTag(40); + output.WriteEnum((int) TestCategory); + } + if (jspbEncodingOptions_ != null) { + output.WriteRawTag(50); + output.WriteMessage(JspbEncodingOptions); + } + if (payloadCase_ == PayloadOneofCase.JspbPayload) { + output.WriteRawTag(58); + output.WriteString(JspbPayload); + } + if (payloadCase_ == PayloadOneofCase.TextPayload) { + output.WriteRawTag(66); + output.WriteString(TextPayload); + } + if (PrintUnknownFields != false) { + output.WriteRawTag(72); + output.WriteBool(PrintUnknownFields); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { + output.WriteRawTag(10); + output.WriteBytes(ProtobufPayload); + } + if (payloadCase_ == PayloadOneofCase.JsonPayload) { + output.WriteRawTag(18); + output.WriteString(JsonPayload); + } + if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { + output.WriteRawTag(24); + output.WriteEnum((int) RequestedOutputFormat); + } + if (MessageType.Length != 0) { + output.WriteRawTag(34); + output.WriteString(MessageType); + } + if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { + output.WriteRawTag(40); + output.WriteEnum((int) TestCategory); + } + if (jspbEncodingOptions_ != null) { + output.WriteRawTag(50); + output.WriteMessage(JspbEncodingOptions); + } + if (payloadCase_ == PayloadOneofCase.JspbPayload) { + output.WriteRawTag(58); + output.WriteString(JspbPayload); + } + if (payloadCase_ == PayloadOneofCase.TextPayload) { + output.WriteRawTag(66); + output.WriteString(TextPayload); + } + if (PrintUnknownFields != false) { + output.WriteRawTag(72); + output.WriteBool(PrintUnknownFields); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (payloadCase_ == PayloadOneofCase.ProtobufPayload) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(ProtobufPayload); + } + if (payloadCase_ == PayloadOneofCase.JsonPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(JsonPayload); + } + if (payloadCase_ == PayloadOneofCase.JspbPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(JspbPayload); + } + if (payloadCase_ == PayloadOneofCase.TextPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(TextPayload); + } + if (RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { + size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) RequestedOutputFormat); + } + if (MessageType.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(MessageType); + } + if (TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { + size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) TestCategory); + } + if (jspbEncodingOptions_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(JspbEncodingOptions); + } + if (PrintUnknownFields != false) { + size += 1 + 1; + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(ConformanceRequest other) { + if (other == null) { + return; + } + if (other.RequestedOutputFormat != global::Conformance.WireFormat.Unspecified) { + RequestedOutputFormat = other.RequestedOutputFormat; + } + if (other.MessageType.Length != 0) { + MessageType = other.MessageType; + } + if (other.TestCategory != global::Conformance.TestCategory.UnspecifiedTest) { + TestCategory = other.TestCategory; + } + if (other.jspbEncodingOptions_ != null) { + if (jspbEncodingOptions_ == null) { + JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); + } + JspbEncodingOptions.MergeFrom(other.JspbEncodingOptions); + } + if (other.PrintUnknownFields != false) { + PrintUnknownFields = other.PrintUnknownFields; + } + switch (other.PayloadCase) { + case PayloadOneofCase.ProtobufPayload: + ProtobufPayload = other.ProtobufPayload; + break; + case PayloadOneofCase.JsonPayload: + JsonPayload = other.JsonPayload; + break; + case PayloadOneofCase.JspbPayload: + JspbPayload = other.JspbPayload; + break; + case PayloadOneofCase.TextPayload: + TextPayload = other.TextPayload; + break; + } + + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + ProtobufPayload = input.ReadBytes(); + break; + } + case 18: { + JsonPayload = input.ReadString(); + break; + } + case 24: { + RequestedOutputFormat = (global::Conformance.WireFormat) input.ReadEnum(); + break; + } + case 34: { + MessageType = input.ReadString(); + break; + } + case 40: { + TestCategory = (global::Conformance.TestCategory) input.ReadEnum(); + break; + } + case 50: { + if (jspbEncodingOptions_ == null) { + JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); + } + input.ReadMessage(JspbEncodingOptions); + break; + } + case 58: { + JspbPayload = input.ReadString(); + break; + } + case 66: { + TextPayload = input.ReadString(); + break; + } + case 72: { + PrintUnknownFields = input.ReadBool(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + ProtobufPayload = input.ReadBytes(); + break; + } + case 18: { + JsonPayload = input.ReadString(); + break; + } + case 24: { + RequestedOutputFormat = (global::Conformance.WireFormat) input.ReadEnum(); + break; + } + case 34: { + MessageType = input.ReadString(); + break; + } + case 40: { + TestCategory = (global::Conformance.TestCategory) input.ReadEnum(); + break; + } + case 50: { + if (jspbEncodingOptions_ == null) { + JspbEncodingOptions = new global::Conformance.JspbEncodingConfig(); + } + input.ReadMessage(JspbEncodingOptions); + break; + } + case 58: { + JspbPayload = input.ReadString(); + break; + } + case 66: { + TextPayload = input.ReadString(); + break; + } + case 72: { + PrintUnknownFields = input.ReadBool(); + break; + } + } + } + } + #endif + + } + + /// + /// Represents a single test case's output. + /// + public sealed partial class ConformanceResponse : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new ConformanceResponse()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceResponse() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceResponse(ConformanceResponse other) : this() { + switch (other.ResultCase) { + case ResultOneofCase.ParseError: + ParseError = other.ParseError; + break; + case ResultOneofCase.SerializeError: + SerializeError = other.SerializeError; + break; + case ResultOneofCase.TimeoutError: + TimeoutError = other.TimeoutError; + break; + case ResultOneofCase.RuntimeError: + RuntimeError = other.RuntimeError; + break; + case ResultOneofCase.ProtobufPayload: + ProtobufPayload = other.ProtobufPayload; + break; + case ResultOneofCase.JsonPayload: + JsonPayload = other.JsonPayload; + break; + case ResultOneofCase.Skipped: + Skipped = other.Skipped; + break; + case ResultOneofCase.JspbPayload: + JspbPayload = other.JspbPayload; + break; + case ResultOneofCase.TextPayload: + TextPayload = other.TextPayload; + break; + } + + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ConformanceResponse Clone() { + return new ConformanceResponse(this); + } + + /// Field number for the "parse_error" field. + public const int ParseErrorFieldNumber = 1; + /// + /// This string should be set to indicate parsing failed. The string can + /// provide more information about the parse error if it is available. + /// + /// Setting this string does not necessarily mean the testee failed the + /// test. Some of the test cases are intentionally invalid input. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string ParseError { + get { return resultCase_ == ResultOneofCase.ParseError ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.ParseError; + } + } + + /// Field number for the "serialize_error" field. + public const int SerializeErrorFieldNumber = 6; + /// + /// If the input was successfully parsed but errors occurred when + /// serializing it to the requested output format, set the error message in + /// this field. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string SerializeError { + get { return resultCase_ == ResultOneofCase.SerializeError ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.SerializeError; + } + } + + /// Field number for the "timeout_error" field. + public const int TimeoutErrorFieldNumber = 9; + /// + /// This should be set if the test program timed out. The string should + /// provide more information about what the child process was doing when it + /// was killed. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string TimeoutError { + get { return resultCase_ == ResultOneofCase.TimeoutError ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.TimeoutError; + } + } + + /// Field number for the "runtime_error" field. + public const int RuntimeErrorFieldNumber = 2; + /// + /// This should be set if some other error occurred. This will always + /// indicate that the test failed. The string can provide more information + /// about the failure. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string RuntimeError { + get { return resultCase_ == ResultOneofCase.RuntimeError ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.RuntimeError; + } + } + + /// Field number for the "protobuf_payload" field. + public const int ProtobufPayloadFieldNumber = 3; + /// + /// If the input was successfully parsed and the requested output was + /// protobuf, serialize it to protobuf and set it in this field. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString ProtobufPayload { + get { return resultCase_ == ResultOneofCase.ProtobufPayload ? (pb::ByteString) result_ : pb::ByteString.Empty; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.ProtobufPayload; + } + } + + /// Field number for the "json_payload" field. + public const int JsonPayloadFieldNumber = 4; + /// + /// If the input was successfully parsed and the requested output was JSON, + /// serialize to JSON and set it in this field. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string JsonPayload { + get { return resultCase_ == ResultOneofCase.JsonPayload ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.JsonPayload; + } + } + + /// Field number for the "skipped" field. + public const int SkippedFieldNumber = 5; + /// + /// For when the testee skipped the test, likely because a certain feature + /// wasn't supported, like JSON input/output. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Skipped { + get { return resultCase_ == ResultOneofCase.Skipped ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.Skipped; + } + } + + /// Field number for the "jspb_payload" field. + public const int JspbPayloadFieldNumber = 7; + /// + /// If the input was successfully parsed and the requested output was JSPB, + /// serialize to JSPB and set it in this field. JSPB is only used inside + /// Google. Opensource testees can just skip it. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string JspbPayload { + get { return resultCase_ == ResultOneofCase.JspbPayload ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.JspbPayload; + } + } + + /// Field number for the "text_payload" field. + public const int TextPayloadFieldNumber = 8; + /// + /// If the input was successfully parsed and the requested output was + /// TEXT_FORMAT, serialize to TEXT_FORMAT and set it in this field. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string TextPayload { + get { return resultCase_ == ResultOneofCase.TextPayload ? (string) result_ : ""; } + set { + result_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + resultCase_ = ResultOneofCase.TextPayload; + } + } + + private object result_; + /// Enum of possible cases for the "result" oneof. + public enum ResultOneofCase { + None = 0, + ParseError = 1, + SerializeError = 6, + TimeoutError = 9, + RuntimeError = 2, + ProtobufPayload = 3, + JsonPayload = 4, + Skipped = 5, + JspbPayload = 7, + TextPayload = 8, + } + private ResultOneofCase resultCase_ = ResultOneofCase.None; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ResultOneofCase ResultCase { + get { return resultCase_; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearResult() { + resultCase_ = ResultOneofCase.None; + result_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as ConformanceResponse); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(ConformanceResponse other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (ParseError != other.ParseError) return false; + if (SerializeError != other.SerializeError) return false; + if (TimeoutError != other.TimeoutError) return false; + if (RuntimeError != other.RuntimeError) return false; + if (ProtobufPayload != other.ProtobufPayload) return false; + if (JsonPayload != other.JsonPayload) return false; + if (Skipped != other.Skipped) return false; + if (JspbPayload != other.JspbPayload) return false; + if (TextPayload != other.TextPayload) return false; + if (ResultCase != other.ResultCase) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (resultCase_ == ResultOneofCase.ParseError) hash ^= ParseError.GetHashCode(); + if (resultCase_ == ResultOneofCase.SerializeError) hash ^= SerializeError.GetHashCode(); + if (resultCase_ == ResultOneofCase.TimeoutError) hash ^= TimeoutError.GetHashCode(); + if (resultCase_ == ResultOneofCase.RuntimeError) hash ^= RuntimeError.GetHashCode(); + if (resultCase_ == ResultOneofCase.ProtobufPayload) hash ^= ProtobufPayload.GetHashCode(); + if (resultCase_ == ResultOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); + if (resultCase_ == ResultOneofCase.Skipped) hash ^= Skipped.GetHashCode(); + if (resultCase_ == ResultOneofCase.JspbPayload) hash ^= JspbPayload.GetHashCode(); + if (resultCase_ == ResultOneofCase.TextPayload) hash ^= TextPayload.GetHashCode(); + hash ^= (int) resultCase_; + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (resultCase_ == ResultOneofCase.ParseError) { + output.WriteRawTag(10); + output.WriteString(ParseError); + } + if (resultCase_ == ResultOneofCase.RuntimeError) { + output.WriteRawTag(18); + output.WriteString(RuntimeError); + } + if (resultCase_ == ResultOneofCase.ProtobufPayload) { + output.WriteRawTag(26); + output.WriteBytes(ProtobufPayload); + } + if (resultCase_ == ResultOneofCase.JsonPayload) { + output.WriteRawTag(34); + output.WriteString(JsonPayload); + } + if (resultCase_ == ResultOneofCase.Skipped) { + output.WriteRawTag(42); + output.WriteString(Skipped); + } + if (resultCase_ == ResultOneofCase.SerializeError) { + output.WriteRawTag(50); + output.WriteString(SerializeError); + } + if (resultCase_ == ResultOneofCase.JspbPayload) { + output.WriteRawTag(58); + output.WriteString(JspbPayload); + } + if (resultCase_ == ResultOneofCase.TextPayload) { + output.WriteRawTag(66); + output.WriteString(TextPayload); + } + if (resultCase_ == ResultOneofCase.TimeoutError) { + output.WriteRawTag(74); + output.WriteString(TimeoutError); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (resultCase_ == ResultOneofCase.ParseError) { + output.WriteRawTag(10); + output.WriteString(ParseError); + } + if (resultCase_ == ResultOneofCase.RuntimeError) { + output.WriteRawTag(18); + output.WriteString(RuntimeError); + } + if (resultCase_ == ResultOneofCase.ProtobufPayload) { + output.WriteRawTag(26); + output.WriteBytes(ProtobufPayload); + } + if (resultCase_ == ResultOneofCase.JsonPayload) { + output.WriteRawTag(34); + output.WriteString(JsonPayload); + } + if (resultCase_ == ResultOneofCase.Skipped) { + output.WriteRawTag(42); + output.WriteString(Skipped); + } + if (resultCase_ == ResultOneofCase.SerializeError) { + output.WriteRawTag(50); + output.WriteString(SerializeError); + } + if (resultCase_ == ResultOneofCase.JspbPayload) { + output.WriteRawTag(58); + output.WriteString(JspbPayload); + } + if (resultCase_ == ResultOneofCase.TextPayload) { + output.WriteRawTag(66); + output.WriteString(TextPayload); + } + if (resultCase_ == ResultOneofCase.TimeoutError) { + output.WriteRawTag(74); + output.WriteString(TimeoutError); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (resultCase_ == ResultOneofCase.ParseError) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(ParseError); + } + if (resultCase_ == ResultOneofCase.SerializeError) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(SerializeError); + } + if (resultCase_ == ResultOneofCase.TimeoutError) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(TimeoutError); + } + if (resultCase_ == ResultOneofCase.RuntimeError) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(RuntimeError); + } + if (resultCase_ == ResultOneofCase.ProtobufPayload) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(ProtobufPayload); + } + if (resultCase_ == ResultOneofCase.JsonPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(JsonPayload); + } + if (resultCase_ == ResultOneofCase.Skipped) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Skipped); + } + if (resultCase_ == ResultOneofCase.JspbPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(JspbPayload); + } + if (resultCase_ == ResultOneofCase.TextPayload) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(TextPayload); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(ConformanceResponse other) { + if (other == null) { + return; + } + switch (other.ResultCase) { + case ResultOneofCase.ParseError: + ParseError = other.ParseError; + break; + case ResultOneofCase.SerializeError: + SerializeError = other.SerializeError; + break; + case ResultOneofCase.TimeoutError: + TimeoutError = other.TimeoutError; + break; + case ResultOneofCase.RuntimeError: + RuntimeError = other.RuntimeError; + break; + case ResultOneofCase.ProtobufPayload: + ProtobufPayload = other.ProtobufPayload; + break; + case ResultOneofCase.JsonPayload: + JsonPayload = other.JsonPayload; + break; + case ResultOneofCase.Skipped: + Skipped = other.Skipped; + break; + case ResultOneofCase.JspbPayload: + JspbPayload = other.JspbPayload; + break; + case ResultOneofCase.TextPayload: + TextPayload = other.TextPayload; + break; + } + + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + ParseError = input.ReadString(); + break; + } + case 18: { + RuntimeError = input.ReadString(); + break; + } + case 26: { + ProtobufPayload = input.ReadBytes(); + break; + } + case 34: { + JsonPayload = input.ReadString(); + break; + } + case 42: { + Skipped = input.ReadString(); + break; + } + case 50: { + SerializeError = input.ReadString(); + break; + } + case 58: { + JspbPayload = input.ReadString(); + break; + } + case 66: { + TextPayload = input.ReadString(); + break; + } + case 74: { + TimeoutError = input.ReadString(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + ParseError = input.ReadString(); + break; + } + case 18: { + RuntimeError = input.ReadString(); + break; + } + case 26: { + ProtobufPayload = input.ReadBytes(); + break; + } + case 34: { + JsonPayload = input.ReadString(); + break; + } + case 42: { + Skipped = input.ReadString(); + break; + } + case 50: { + SerializeError = input.ReadString(); + break; + } + case 58: { + JspbPayload = input.ReadString(); + break; + } + case 66: { + TextPayload = input.ReadString(); + break; + } + case 74: { + TimeoutError = input.ReadString(); + break; + } + } + } + } + #endif + + } + + /// + /// Encoding options for jspb format. + /// + public sealed partial class JspbEncodingConfig : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new JspbEncodingConfig()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public JspbEncodingConfig() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public JspbEncodingConfig(JspbEncodingConfig other) : this() { + useJspbArrayAnyFormat_ = other.useJspbArrayAnyFormat_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public JspbEncodingConfig Clone() { + return new JspbEncodingConfig(this); + } + + /// Field number for the "use_jspb_array_any_format" field. + public const int UseJspbArrayAnyFormatFieldNumber = 1; + private bool useJspbArrayAnyFormat_; + /// + /// Encode the value field of Any as jspb array if true, otherwise binary. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool UseJspbArrayAnyFormat { + get { return useJspbArrayAnyFormat_; } + set { + useJspbArrayAnyFormat_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as JspbEncodingConfig); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(JspbEncodingConfig other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (UseJspbArrayAnyFormat != other.UseJspbArrayAnyFormat) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (UseJspbArrayAnyFormat != false) hash ^= UseJspbArrayAnyFormat.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (UseJspbArrayAnyFormat != false) { + output.WriteRawTag(8); + output.WriteBool(UseJspbArrayAnyFormat); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (UseJspbArrayAnyFormat != false) { + output.WriteRawTag(8); + output.WriteBool(UseJspbArrayAnyFormat); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (UseJspbArrayAnyFormat != false) { + size += 1 + 1; + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(JspbEncodingConfig other) { + if (other == null) { + return; + } + if (other.UseJspbArrayAnyFormat != false) { + UseJspbArrayAnyFormat = other.UseJspbArrayAnyFormat; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 8: { + UseJspbArrayAnyFormat = input.ReadBool(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 8: { + UseJspbArrayAnyFormat = input.ReadBool(); + break; + } + } + } + } + #endif + + } + + #endregion + +} + +#endregion Designer generated code From 1de344fcd1c1b2c6ec937151d69634171463370d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 Feb 2023 11:02:41 -0800 Subject: [PATCH 035/463] Document known quirks of EnumDescriptor::is_closed() when importing across files with different syntaxes. PiperOrigin-RevId: 509581394 --- .../java/com/google/protobuf/Descriptors.java | 25 +++++++++++++------ python/google/protobuf/descriptor.py | 24 +++++++++++++----- src/google/protobuf/descriptor.h | 14 ++++++++++- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index e6503a05d7ec..e5b973f15804 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -303,9 +303,7 @@ public static FileDescriptor buildFrom(FileDescriptorProto proto, FileDescriptor * two messages were defined with the same name. */ public static FileDescriptor buildFrom( - FileDescriptorProto proto, - FileDescriptor[] dependencies, - boolean allowUnknownDependencies) + FileDescriptorProto proto, FileDescriptor[] dependencies, boolean allowUnknownDependencies) throws DescriptorValidationException { // Building descriptors involves two steps: translating and linking. // In the translation step (implemented by FileDescriptor's @@ -462,9 +460,9 @@ public static FileDescriptor internalBuildGeneratedFileFrom( } /** - * This method is to be called by generated code only. It updates the - * FileDescriptorProto associated with the descriptor by parsing it again with the given - * ExtensionRegistry. This is needed to recognize custom options. + * This method is to be called by generated code only. It updates the FileDescriptorProto + * associated with the descriptor by parsing it again with the given ExtensionRegistry. This is + * needed to recognize custom options. */ public static void internalUpdateFileDescriptor( FileDescriptor descriptor, ExtensionRegistry registry) { @@ -1778,10 +1776,23 @@ public FileDescriptor getFile() { *

Closed enum means that it: * *

    - *
  • Has a fixed set of named values. * + *
  • Has a fixed set of values, rather than being equivalent to an int32. *
  • Encountering values not in this set causes them to be treated as unknown fields. *
  • The first value (i.e., the default) may be nonzero. *
+ * + *

WARNING: Some runtimes currently have a quirk where non-closed enums are treated as closed + * when used as the type of fields defined in a `syntax = proto2;` file. This quirk is not + * present in all runtimes; as of writing, we know that: + * + *

    + *
  • C++, Java, and C++-based Python share this quirk. + *
  • UPB and UPB-based Python do not. + *
  • PHP and Ruby treat all enums as open regardless of declaration. + *
+ * + *

Care should be taken when using this function to respect the target runtime's enum + * handling quirks. */ public boolean isClosed() { return getFile().getSyntax() != Syntax.PROTO3; diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index e5d547a3bbeb..fcb87cab5581 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -739,13 +739,25 @@ def __init__(self, name, full_name, filename, values, @property def is_closed(self): - """If the enum is closed. + """Returns true whether this is a "closed" enum. - closed enum means: - - Has a fixed set of named values. - - Encountering values not in this set causes them to be treated as - unknown fields. - - The first value (i.e., the default) may be nonzero. + This means that it: + - Has a fixed set of values, rather than being equivalent to an int32. + - Encountering values not in this set causes them to be treated as unknown + fields. + - The first value (i.e., the default) may be nonzero. + + WARNING: Some runtimes currently have a quirk where non-closed enums are + treated as closed when used as the type of fields defined in a + `syntax = proto2;` file. This quirk is not present in all runtimes; as of + writing, we know that: + + - C++, Java, and C++-based Python share this quirk. + - UPB and UPB-based Python do not. + - PHP and Ruby treat all enums as open regardless of declaration. + + Care should be taken when using this function to respect the target + runtime's enum handling quirks. """ return self.file.syntax == 'proto2' diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 1caf5c194b8b..f6359e845cc7 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -1151,10 +1151,22 @@ class PROTOBUF_EXPORT EnumDescriptor : private internal::SymbolBase { bool is_placeholder() const; // Returns true whether this is a "closed" enum, meaning that it: - // - Has a fixed set of named values. + // - Has a fixed set of values, rather than being equivalent to an int32. // - Encountering values not in this set causes them to be treated as unknown // fields. // - The first value (i.e., the default) may be nonzero. + // + // WARNING: Some runtimes currently have a quirk where non-closed enums are + // treated as closed when used as the type of fields defined in a + // `syntax = proto2;` file. This quirk is not present in all runtimes; as of + // writing, we know that: + // + // - C++, Java, and C++-based Python share this quirk. + // - UPB and UPB-based Python do not. + // - PHP and Ruby treat all enums as open regardless of declaration. + // + // Care should be taken when using this function to respect the target + // runtime's enum handling quirks. bool is_closed() const; // Reserved fields ------------------------------------------------- From 0c21b63cb7903460696080d0d4fbdfd42525db5e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 14 Feb 2023 13:08:07 -0800 Subject: [PATCH 036/463] Only run code-spell on our main branches PiperOrigin-RevId: 509616839 --- .github/workflows/codespell.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index e17f174d4ec2..26b4a327b461 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -2,7 +2,16 @@ # https://github.com/codespell-project/actions-codespell # https://github.com/codespell-project/codespell name: codespell -on: [push, pull_request] +on: + push: + branches: + - main + - '[0-9]+.x' + pull_request: + branches: + - main + - '[0-9]+.x' + permissions: contents: read # to fetch code (actions/checkout) jobs: From 8f89e0d8db648bc7d530c82afc0d794c1faaf134 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 10:01:58 -0800 Subject: [PATCH 037/463] [ObjC] Enforce bytes and string field size limits. PiperOrigin-RevId: 509863774 --- objectivec/GPBCodedInputStream.m | 36 ++++++++++++++----- objectivec/GPBMessage.m | 3 -- objectivec/Tests/GPBCodedInputStreamTests.m | 20 +++++++++-- .../Tests/GPBMessageTests+Serialization.m | 13 ++++--- 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index ad5c75e4ef88..9acf6f170fca 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -51,6 +51,10 @@ // int CodedInputStream::default_recursion_limit_ = 100; static const NSUInteger kDefaultRecursionLimit = 100; +// Bytes and Strings have a max size of 2GB. +// https://protobuf.dev/programming-guides/encoding/#cheat-sheet +static const uint32_t kMaxFieldSize = 0x7fffffff; + static void RaiseException(NSInteger code, NSString *reason) { NSDictionary *errorInfo = nil; if ([reason length]) { @@ -223,14 +227,20 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { } NSString *GPBCodedInputStreamReadRetainedString(GPBCodedInputStreamState *state) { - int32_t size = ReadRawVarint32(state); + uint64_t size = GPBCodedInputStreamReadUInt64(state); + if (size > kMaxFieldSize) { + // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change + // so reuse an existing one. + RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + } + NSUInteger ns_size = (NSUInteger)size; NSString *result; if (size == 0) { result = @""; } else { CheckSize(state, size); result = [[NSString alloc] initWithBytes:&state->bytes[state->bufferPos] - length:size + length:ns_size encoding:NSUTF8StringEncoding]; state->bufferPos += size; if (!result) { @@ -246,21 +256,31 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { } NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state) { - int32_t size = ReadRawVarint32(state); - if (size < 0) return nil; + uint64_t size = GPBCodedInputStreamReadUInt64(state); + if (size > kMaxFieldSize) { + // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change + // so reuse an existing one. + RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + } + NSUInteger ns_size = (NSUInteger)size; CheckSize(state, size); - NSData *result = [[NSData alloc] initWithBytes:state->bytes + state->bufferPos length:size]; + NSData *result = [[NSData alloc] initWithBytes:state->bytes + state->bufferPos length:ns_size]; state->bufferPos += size; return result; } NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state) { - int32_t size = ReadRawVarint32(state); - if (size < 0) return nil; + uint64_t size = GPBCodedInputStreamReadUInt64(state); + if (size > kMaxFieldSize) { + // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change + // so reuse an existing one. + RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + } + NSUInteger ns_size = (NSUInteger)size; CheckSize(state, size); // Cast is safe because freeWhenDone is NO. NSData *result = [[NSData alloc] initWithBytesNoCopy:(void *)(state->bytes + state->bufferPos) - length:size + length:ns_size freeWhenDone:NO]; state->bufferPos += size; return result; diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index f41457066a44..a06bb6feb8cf 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -2003,9 +2003,6 @@ - (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input return; } NSData *data = GPBCodedInputStreamReadRetainedBytesNoCopy(state); - if (data == nil) { - return; - } [self mergeFromData:data extensionRegistry:extensionRegistry]; [data release]; } diff --git a/objectivec/Tests/GPBCodedInputStreamTests.m b/objectivec/Tests/GPBCodedInputStreamTests.m index 05040625f661..0d0d32734cda 100644 --- a/objectivec/Tests/GPBCodedInputStreamTests.m +++ b/objectivec/Tests/GPBCodedInputStreamTests.m @@ -28,6 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#import #import "GPBTestUtilities.h" #import "GPBCodedInputStream.h" @@ -370,10 +371,23 @@ - (void)testInvalidGroupEndTagThrows { @"should throw a GPBCodedInputStreamException exception "); } -- (void)testBytesWithNegativeSize { - NSData* data = bytes(0xFF, 0xFF, 0xFF, 0xFF, 0x0F); +- (void)testBytesOver2GB { + NSData* data = bytes(0xFF, 0xFF, 0xFF, 0xFF, 0x0F, 0x01, 0x02, 0x03); // don't need all the bytes GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data]; - XCTAssertNil([input readBytes]); + @try { + __unused NSData* result = [input readBytes]; + XCTFail(@"Should have thrown"); + } @catch (NSException* anException) { + // Ensure the correct error within the exception. + XCTAssertTrue([anException isKindOfClass:[NSException class]]); + XCTAssertEqualObjects(anException.name, GPBCodedInputStreamException); + NSDictionary* userInfo = anException.userInfo; + XCTAssertNotNil(userInfo); + NSError* err = userInfo[GPBCodedInputStreamUnderlyingErrorKey]; + XCTAssertNotNil(err); + XCTAssertEqualObjects(err.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(err.code, GPBCodedInputStreamErrorInvalidSize); + } } // Verifies fix for b/10315336. diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index 3c2381c3dea3..5a7dc16c84df 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -1266,12 +1266,17 @@ - (void)testErrorRecursionDepthReached { XCTAssertEqual(error.code, GPBCodedInputStreamErrorRecursionDepthExceeded); } -- (void)testParseDelimitedDataWithNegativeSize { - NSData *data = DataFromCStr("\xFF\xFF\xFF\xFF\x0F"); +- (void)testParseDelimitedDataOver2GB { + NSData *data = DataFromCStr("\xFF\xFF\xFF\xFF\x0F\x01\x02\0x3"); // Don't need all the bytes GPBCodedInputStream *input = [GPBCodedInputStream streamWithData:data]; NSError *error; - [GPBMessage parseDelimitedFromCodedInputStream:input extensionRegistry:nil error:&error]; - XCTAssertNil(error); + GPBMessage *result = [GPBMessage parseDelimitedFromCodedInputStream:input + extensionRegistry:nil + error:&error]; + XCTAssertNil(result); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSize); } #ifdef DEBUG From f5f5a1faf1dcc276a06d98ea51ed69c9c480dca8 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 10:30:37 -0800 Subject: [PATCH 038/463] Annotate remaining field semantics. PiperOrigin-RevId: 509872256 --- .../cpp/field_generators/enum_field.cc | 13 +- .../cpp/field_generators/map_field.cc | 7 +- .../cpp/field_generators/message_field.cc | 6 +- .../cpp/field_generators/primitive_field.cc | 72 ++-- src/google/protobuf/compiler/cpp/message.cc | 6 +- .../protobuf/compiler/cpp/metadata_test.cc | 321 +++++++++++++++++- 6 files changed, 365 insertions(+), 60 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc index 9744204030db..19bd6b58e4f8 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc @@ -51,6 +51,7 @@ namespace protobuf { namespace compiler { namespace cpp { namespace { +using Semantic = ::google::protobuf::io::AnnotationCollector::Semantic; using Sub = ::google::protobuf::io::Printer::Sub; std::vector Vars(const FieldDescriptor* field, const Options& opts) { @@ -172,7 +173,8 @@ class SingularEnum : public FieldGeneratorBase { void SingularEnum::GenerateAccessorDeclarations(io::Printer* p) const { auto v = p->WithVars( - AnnotatedAccessors(field_, {"", "set_", "_internal_", "_internal_set_"})); + AnnotatedAccessors(field_, {"", "_internal_", "_internal_set_"})); + auto vs = p->WithVars(AnnotatedAccessors(field_, {"set_"}, Semantic::kSet)); p->Emit(R"cc( $DEPRECATED$ $Enum$ $name$() const; $DEPRECATED$ void $set_name$($Enum$ value); @@ -331,9 +333,12 @@ class RepeatedEnum : public FieldGeneratorBase { }; void RepeatedEnum::GenerateAccessorDeclarations(io::Printer* p) const { - auto v = p->WithVars( - AnnotatedAccessors(field_, {"", "set_", "add_", "mutable_", "_internal_", - "_internal_add_", "_internal_mutable_"})); + auto v = p->WithVars(AnnotatedAccessors( + field_, {"", "_internal_", "_internal_add_", "_internal_mutable_"})); + auto vs = + p->WithVars(AnnotatedAccessors(field_, {"set_", "add_"}, Semantic::kSet)); + auto vm = + p->WithVars(AnnotatedAccessors(field_, {"mutable_"}, Semantic::kAlias)); p->Emit(R"cc( public: diff --git a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc index 79f1015c68be..a91a3947e709 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc @@ -30,6 +30,7 @@ #include #include +#include #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" @@ -141,10 +142,12 @@ void MapFieldGenerator::GenerateAccessorDeclarations( " ${1$_internal_mutable_$name$$}$();\n" "public:\n" "$deprecated_attr$const ::$proto_ns$::Map< $key_cpp$, $val_cpp$ >&\n" - " ${1$$name$$}$() const;\n" + " ${1$$name$$}$() const;\n", + descriptor_); + format( "$deprecated_attr$::$proto_ns$::Map< $key_cpp$, $val_cpp$ >*\n" " ${1$mutable_$name$$}$();\n", - descriptor_); + std::make_tuple(descriptor_, GeneratedCodeInfo::Annotation::ALIAS)); } void MapFieldGenerator::GenerateInlineAccessorDefinitions( diff --git a/src/google/protobuf/compiler/cpp/field_generators/message_field.cc b/src/google/protobuf/compiler/cpp/field_generators/message_field.cc index 6eaadefefd6b..b4bf2a91a824 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/message_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/message_field.cc @@ -809,9 +809,11 @@ void RepeatedMessageFieldGenerator::GenerateAccessorDeclarations( "$type$* ${1$_internal_add_$name$$}$();\n" "public:\n", descriptor_); + format("$deprecated_attr$const $type$& ${1$$name$$}$(int index) const;\n", + descriptor_); + format("$deprecated_attr$$type$* ${1$add_$name$$}$();\n", + std::make_tuple(descriptor_, GeneratedCodeInfo::Annotation::SET)); format( - "$deprecated_attr$const $type$& ${1$$name$$}$(int index) const;\n" - "$deprecated_attr$$type$* ${1$add_$name$$}$();\n" "$deprecated_attr$const ::$proto_ns$::RepeatedPtrField< $type$ >&\n" " ${1$$name$$}$() const;\n", descriptor_); diff --git a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc index 5bbf298341c4..b15801ea51f8 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc @@ -56,7 +56,7 @@ namespace { using ::google::protobuf::internal::WireFormat; using ::google::protobuf::internal::WireFormatLite; using Sub = ::google::protobuf::io::Printer::Sub; -using Annotation = ::google::protobuf::GeneratedCodeInfo::Annotation; +using Semantic = ::google::protobuf::io::AnnotationCollector::Semantic; // For encodings with fixed sizes, returns that size in bytes. absl::optional FixedSize(FieldDescriptor::Type type) { @@ -193,18 +193,10 @@ class SingularPrimitive final : public FieldGeneratorBase { }; void SingularPrimitive::GenerateAccessorDeclarations(io::Printer* p) const { + auto v = p->WithVars( + AnnotatedAccessors(field_, {"", "_internal_", "_internal_set_"})); + auto vs = p->WithVars(AnnotatedAccessors(field_, {"set_"}, Semantic::kSet)); p->Emit( - { - Sub("name", p->LookupVar("name")).AnnotatedAs(field_), - Sub("set_name", absl::StrCat("set_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("_internal_name", - absl::StrCat("_internal_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("_internal_set_name", - absl::StrCat("_internal_set_", p->LookupVar("name"))) - .AnnotatedAs(field_), - }, R"cc( $DEPRECATED$ $Type$ $name$() const; $DEPRECATED$ void $set_name$($Type$ value); @@ -397,41 +389,27 @@ void RepeatedPrimitive::GeneratePrivateMembers(io::Printer* p) const { } void RepeatedPrimitive::GenerateAccessorDeclarations(io::Printer* p) const { - p->Emit( - { - Sub("name", p->LookupVar("name")).AnnotatedAs(field_), - Sub("set_name", absl::StrCat("set_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("add_name", absl::StrCat("add_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("mutable_name", absl::StrCat("mutable_", p->LookupVar("name"))) - .AnnotatedAs(field_), - - Sub("_internal_name", - absl::StrCat("_internal_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("_internal_add_name", - absl::StrCat("_internal_add_", p->LookupVar("name"))) - .AnnotatedAs(field_), - Sub("_internal_mutable_name", - absl::StrCat("_internal_mutable_", p->LookupVar("name"))) - .AnnotatedAs(field_), - }, - R"cc( - $DEPRECATED$ $Type$ $name$(int index) const; - $DEPRECATED$ void $set_name$(int index, $Type$ value); - $DEPRECATED$ void $add_name$($Type$ value); - $DEPRECATED$ const $pb$::RepeatedField<$Type$>& $name$() const; - $DEPRECATED$ $pb$::RepeatedField<$Type$>* $mutable_name$(); - - private: - $Type$ $_internal_name$(int index) const; - void $_internal_add_name$($Type$ value); - const $pb$::RepeatedField<$Type$>& $_internal_name$() const; - $pb$::RepeatedField<$Type$>* $_internal_mutable_name$(); - - public: - )cc"); + auto v = p->WithVars(AnnotatedAccessors( + field_, {"", "_internal_", "_internal_add_", "_internal_mutable_"})); + auto vs = + p->WithVars(AnnotatedAccessors(field_, {"set_", "add_"}, Semantic::kSet)); + auto va = + p->WithVars(AnnotatedAccessors(field_, {"mutable_"}, Semantic::kAlias)); + p->Emit(R"cc( + $DEPRECATED$ $Type$ $name$(int index) const; + $DEPRECATED$ void $set_name$(int index, $Type$ value); + $DEPRECATED$ void $add_name$($Type$ value); + $DEPRECATED$ const $pb$::RepeatedField<$Type$>& $name$() const; + $DEPRECATED$ $pb$::RepeatedField<$Type$>* $mutable_name$(); + + private: + $Type$ $_internal_name$(int index) const; + void $_internal_add_name$($Type$ value); + const $pb$::RepeatedField<$Type$>& $_internal_name$() const; + $pb$::RepeatedField<$Type$>* $_internal_mutable_name$(); + + public: + )cc"); } void RepeatedPrimitive::GenerateInlineAccessorDefinitions( diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 88bb01551877..a9cb0545d0ff 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -85,6 +85,7 @@ using ::google::protobuf::internal::WireFormat; using ::google::protobuf::internal::WireFormatLite; using ::google::protobuf::internal::cpp::HasHasbit; using ::google::protobuf::internal::cpp::Utf8CheckMode; +using Semantic = ::google::protobuf::io::AnnotationCollector::Semantic; using Sub = ::google::protobuf::io::Printer::Sub; static constexpr int kNoHasbit = -1; @@ -730,7 +731,10 @@ void MessageGenerator::GenerateFieldAccessorDeclarations(io::Printer* p) { {"clearer", [&] { p->Emit({Sub("clear_name", absl::StrCat("clear_", name)) - .AnnotatedAs(field)}, + .AnnotatedAs({ + field, + Semantic::kSet, + })}, R"cc( $deprecated_attr $void $clear_name$() $impl$; )cc"); diff --git a/src/google/protobuf/compiler/cpp/metadata_test.cc b/src/google/protobuf/compiler/cpp/metadata_test.cc index feb5a7f66c88..e95ee566f81b 100644 --- a/src/google/protobuf/compiler/cpp/metadata_test.cc +++ b/src/google/protobuf/compiler/cpp/metadata_test.cc @@ -177,6 +177,270 @@ TEST_F(CppMetadataTest, RangeChecksWork) { EXPECT_FALSE(atu::GetAnnotationSubstring(test, annotation).has_value()); } +constexpr absl::string_view kEnumFieldTestFile = R"( + syntax = "proto2"; + package foo; + enum Enum { VALUE = 0; } + message Message { + optional Enum efield = 1; + repeated Enum refield = 2; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesEnumSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kEnumFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector field_path{FileDescriptorProto::kMessageTypeFieldNumber, 0, + DescriptorProto::kFieldFieldNumber, 0}; + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "efield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_efield" || *substring == "clear_efield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } + } + field_path.back() = 1; + annotations.clear(); + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "refield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_refield" || *substring == "clear_refield" || + *substring == "add_refield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_refield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } +} + +constexpr absl::string_view kMapFieldTestFile = R"( + syntax = "proto2"; + package foo; + message Message { + map mfield = 1; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesMapSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kMapFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector field_path{FileDescriptorProto::kMessageTypeFieldNumber, 0, + DescriptorProto::kFieldFieldNumber, 0}; + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "clear_mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } +} + +constexpr absl::string_view kPrimFieldTestFile = R"( + syntax = "proto2"; + package foo; + message Message { + optional int32 ifield = 1; + repeated int32 rifield = 2; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesPrimSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kPrimFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector field_path{FileDescriptorProto::kMessageTypeFieldNumber, 0, + DescriptorProto::kFieldFieldNumber, 0}; + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "ifield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_ifield" || *substring == "clear_ifield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } + } + field_path.back() = 1; + annotations.clear(); + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "rifield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_rifield" || *substring == "clear_rifield" || + *substring == "add_rifield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_rifield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } +} + +constexpr absl::string_view kCordFieldTestFile = R"( + syntax = "proto2"; + package foo; + message Message { + optional string sfield = 1 [ctype = CORD]; + repeated string rsfield = 2 [ctype = CORD]; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesCordSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kCordFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector field_path{FileDescriptorProto::kMessageTypeFieldNumber, 0, + DescriptorProto::kFieldFieldNumber, 0}; + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_sfield" || *substring == "clear_sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } + field_path.back() = 1; + annotations.clear(); + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_rsfield" || *substring == "clear_rsfield" || + *substring == "add_rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } +} + +constexpr absl::string_view kStringPieceFieldTestFile = R"( + syntax = "proto2"; + package foo; + message Message { + optional string sfield = 1 [ctype = STRING_PIECE]; + repeated string rsfield = 2 [ctype = STRING_PIECE]; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesStringPieceSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kStringPieceFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector field_path{FileDescriptorProto::kMessageTypeFieldNumber, 0, + DescriptorProto::kFieldFieldNumber, 0}; + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_sfield" || *substring == "set_alias_sfield" || + *substring == "clear_sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_sfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } + field_path.back() = 1; + annotations.clear(); + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "set_rsfield" || + *substring == "set_alias_rsfield" || + *substring == "clear_rsfield" || + *substring == "add_alias_rsfield" || + *substring == "add_rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_rsfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } + } +} + constexpr absl::string_view kStringFieldTestFile = R"( syntax = "proto2"; package foo; @@ -205,7 +469,7 @@ TEST_F(CppMetadataTest, AnnotatesStringSemantics) { if (*substring == "sfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, annotation->semantic()); - } else if (*substring == "set_sfield") { + } else if (*substring == "set_sfield" || *substring == "clear_sfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, annotation->semantic()); } else if (*substring == "mutable_sfield") { @@ -223,7 +487,8 @@ TEST_F(CppMetadataTest, AnnotatesStringSemantics) { if (*substring == "rsfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, annotation->semantic()); - } else if (*substring == "set_rsfield") { + } else if (*substring == "set_rsfield" || *substring == "clear_rsfield" || + *substring == "add_rsfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, annotation->semantic()); } else if (*substring == "mutable_rsfield") { @@ -265,6 +530,9 @@ TEST_F(CppMetadataTest, AnnotatesMessageSemantics) { if (*substring == "mfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, annotation->semantic()); + } else if (*substring == "clear_mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); } else if (*substring == "mutable_mfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, annotation->semantic()); @@ -277,15 +545,60 @@ TEST_F(CppMetadataTest, AnnotatesMessageSemantics) { for (const auto* annotation : annotations) { auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); ASSERT_TRUE(substring.has_value()); - if (substring == "rmfield") { + if (*substring == "rmfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, annotation->semantic()); - } else if (substring == "mutable_rmfield") { + } else if (*substring == "add_rmfield" || *substring == "clear_rmfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } else if (*substring == "mutable_rmfield") { EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, annotation->semantic()); } } } + +constexpr absl::string_view kLazyMessageFieldTestFile = R"( + syntax = "proto2"; + package foo; + message SMessage { } + message Message { + optional SMessage mfield = 1 [lazy=true]; + } +)"; + +TEST_F(CppMetadataTest, AnnotatesLazyMessageSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string pb_h; + atu::AddFile("test.proto", kLazyMessageFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", &file, &pb_h, &info, nullptr, + nullptr, nullptr)); + EXPECT_EQ("Message", file.message_type(1).name()); + std::vector field_path; + field_path.push_back(FileDescriptorProto::kMessageTypeFieldNumber); + field_path.push_back(1); + field_path.push_back(DescriptorProto::kFieldFieldNumber); + field_path.push_back(0); + std::vector annotations; + atu::FindAnnotationsOnPath(info, "test.proto", field_path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(pb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (*substring == "mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_NONE, + annotation->semantic()); + } else if (*substring == "mutable_mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_ALIAS, + annotation->semantic()); + } else if (*substring == "set_encoded_mfield" || + *substring == "clear_mfield") { + EXPECT_EQ(GeneratedCodeInfo_Annotation_Semantic_SET, + annotation->semantic()); + } + } +} } // namespace } // namespace cpp } // namespace compiler From 0c013ec412468e21ea68b234935e7490eb28f8ce Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 11:45:00 -0800 Subject: [PATCH 039/463] [ObjC] Avoid data copy within parseDelimitedFromCodedInputStream:... PiperOrigin-RevId: 509893038 --- objectivec/GPBMessage.m | 40 ++++++------------- objectivec/GPBMessage_PackagePrivate.h | 5 --- .../Tests/GPBMessageTests+Serialization.m | 2 +- 3 files changed, 14 insertions(+), 33 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index a06bb6feb8cf..1c1c8f80731f 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -1994,19 +1994,6 @@ - (void)mergeFromData:(NSData *)data extensionRegistry:(id [input release]; } -#pragma mark - mergeDelimitedFrom - -- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input - extensionRegistry:(id)extensionRegistry { - GPBCodedInputStreamState *state = &input->state_; - if (GPBCodedInputStreamIsAtEnd(state)) { - return; - } - NSData *data = GPBCodedInputStreamReadRetainedBytesNoCopy(state); - [self mergeFromData:data extensionRegistry:extensionRegistry]; - [data release]; -} - #pragma mark - Parse From Data Support + (instancetype)parseFromData:(NSData *)data error:(NSError **)errorPtr { @@ -2033,27 +2020,26 @@ + (instancetype)parseFromCodedInputStream:(GPBCodedInputStream *)input + (instancetype)parseDelimitedFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { - GPBMessage *message = [[[self alloc] init] autorelease]; + GPBMessage *result = nil; @try { - [message mergeDelimitedFromCodedInputStream:input extensionRegistry:extensionRegistry]; - if (errorPtr) { - *errorPtr = nil; + uint64_t size = GPBCodedInputStreamReadUInt64(&input->state_); + size_t oldLimit = [input pushLimit:size]; + result = [self parseFromCodedInputStream:input + extensionRegistry:extensionRegistry + error:errorPtr]; + if (result) { + [input popLimit:oldLimit]; + if (errorPtr) { + *errorPtr = nil; + } } } @catch (NSException *exception) { - message = nil; + result = nil; if (errorPtr) { *errorPtr = ErrorFromException(exception); } } -#ifdef DEBUG - if (message && !message.initialized) { - message = nil; - if (errorPtr) { - *errorPtr = MessageError(GPBMessageErrorCodeMissingRequiredField, nil); - } - } -#endif - return message; + return result; } #pragma mark - Unknown Field Support diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 70b47a59bc88..1b3a99332df8 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -80,11 +80,6 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry; -// Parses the next delimited message of this type from the input and merges it -// with this message. -- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input - extensionRegistry:(id)extensionRegistry; - - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data; @end diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index 5a7dc16c84df..68b62ef7acf8 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -1276,7 +1276,7 @@ - (void)testParseDelimitedDataOver2GB { XCTAssertNil(result); XCTAssertNotNil(error); XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); - XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSize); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSubsectionLimit); } #ifdef DEBUG From 335a0c58b756711d390092f124f4a8ff3bb6627b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 12:37:55 -0800 Subject: [PATCH 040/463] Internal changes to extensions PiperOrigin-RevId: 509903560 --- src/google/protobuf/descriptor.cc | 1 + src/google/protobuf/descriptor.h | 3 ++- src/google/protobuf/descriptor_unittest.cc | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 9e3a49d7bfbb..efcd86e35895 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -5375,6 +5375,7 @@ struct IncrementWhenDestroyed { } // namespace + void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, const Descriptor* parent, Descriptor* result, diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index f6359e845cc7..22bc03c74586 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -65,6 +65,7 @@ #include "google/protobuf/port.h" #include "absl/base/attributes.h" #include "absl/base/call_once.h" +#include "absl/container/btree_map.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" @@ -1902,7 +1903,7 @@ class PROTOBUF_EXPORT DescriptorPool { // in a .proto file. enum ErrorLocation { NAME, // the symbol name, or the package name for files - NUMBER, // field or extension range number + NUMBER, // field, extension range or extension decl number TYPE, // field type EXTENDEE, // field extendee DEFAULT_VALUE, // field default value diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 80eebfec8443..64e6b2c52ad7 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -3979,6 +3979,16 @@ class ValidationErrorTest : public testing::Test { BuildFileInTestPool(DescriptorProto::descriptor()->file()); } + void BuildDescriptorMessagesInTestPoolWithErrors( + absl::string_view expected_errors) { + FileDescriptorProto file_proto; + DescriptorProto::descriptor()->file()->CopyTo(&file_proto); + MockErrorCollector error_collector; + EXPECT_TRUE(pool_.BuildFileCollectingErrors(file_proto, &error_collector) == + nullptr); + EXPECT_EQ(error_collector.text_, expected_errors); + } + DescriptorPool pool_; }; From 4b1cd0dbe38d6288e939a511b7fca232960bcad0 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 13:24:29 -0800 Subject: [PATCH 041/463] [ObjC] Update comments about common failure causes. PiperOrigin-RevId: 509914968 --- objectivec/GPBMessage.h | 29 +++++++++++++++++++++++++++++ objectivec/GPBMessage.m | 18 ++++++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/objectivec/GPBMessage.h b/objectivec/GPBMessage.h index 5b7ce0cb4a49..d9fc8986a2b7 100644 --- a/objectivec/GPBMessage.h +++ b/objectivec/GPBMessage.h @@ -293,6 +293,10 @@ CF_EXTERN_C_END * * @note This can raise the GPBCodedOutputStreamException_* exceptions. * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. **/ - (void)writeToCodedOutputStream:(GPBCodedOutputStream *)output; @@ -302,6 +306,11 @@ CF_EXTERN_C_END * @param output The output stream into which to write the message. * * @note This can raise the GPBCodedOutputStreamException_* exceptions. + * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. **/ - (void)writeToOutputStream:(NSOutputStream *)output; @@ -312,6 +321,11 @@ CF_EXTERN_C_END * @param output The coded output stream into which to write the message. * * @note This can raise the GPBCodedOutputStreamException_* exceptions. + * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. **/ - (void)writeDelimitedToCodedOutputStream:(GPBCodedOutputStream *)output; @@ -322,6 +336,11 @@ CF_EXTERN_C_END * @param output The output stream into which to write the message. * * @note This can raise the GPBCodedOutputStreamException_* exceptions. + * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. **/ - (void)writeDelimitedToOutputStream:(NSOutputStream *)output; @@ -336,6 +355,11 @@ CF_EXTERN_C_END * @note In DEBUG ONLY, the message is also checked for all required field, * if one is missing, nil will be returned. * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. + * * @return The binary representation of the message. **/ - (nullable NSData *)data; @@ -347,6 +371,11 @@ CF_EXTERN_C_END * @note This value is not cached, so if you are using it repeatedly, it is * recommended to keep a local copy. * + * @note The most common cause of this failing is from one thread calling this + * while another thread has a reference to this message or a message used + * within a field and that other thread mutating the message while this + * serialization is taking place. + * * @return The binary representation of the size along with the message. **/ - (NSData *)delimitedData; diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 1c1c8f80731f..84953476d146 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -1308,9 +1308,12 @@ - (NSData *)data { @try { [self writeToCodedOutputStream:stream]; } @catch (NSException *exception) { - // This really shouldn't happen. The only way writeToCodedOutputStream: - // could throw is if something in the library has a bug and the - // serializedSize was wrong. + // This really shouldn't happen. Normally, this could mean there was a bug in the library and it + // failed to match between computing the size and writing out the bytes. However, the more + // common cause is while one thread was writing out the data, some other thread had a reference + // to this message or a message used as a nested field, and that other thread mutated that + // message, causing the pre computed serializedSize to no longer match the final size after + // serialization. It is not safe to mutate a message while accessing it from another thread. #ifdef DEBUG NSLog(@"%@: Internal exception while building message data: %@", [self class], exception); #endif @@ -1328,9 +1331,12 @@ - (NSData *)delimitedData { @try { [self writeDelimitedToCodedOutputStream:stream]; } @catch (NSException *exception) { - // This really shouldn't happen. The only way writeToCodedOutputStream: - // could throw is if something in the library has a bug and the - // serializedSize was wrong. + // This really shouldn't happen. Normally, this could mean there was a bug in the library and it + // failed to match between computing the size and writing out the bytes. However, the more + // common cause is while one thread was writing out the data, some other thread had a reference + // to this message or a message used as a nested field, and that other thread mutated that + // message, causing the pre computed serializedSize to no longer match the final size after + // serialization. It is not safe to mutate a message while accessing it from another thread. #ifdef DEBUG NSLog(@"%@: Internal exception while building message delimitedData: %@", [self class], exception); From fc272edc0a687e2344752e9c023cece292ffa836 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 15 Feb 2023 14:14:29 -0800 Subject: [PATCH 042/463] Automated rollback of commit 1cf2901a1d428fb5101e35b7ac3a44be57226108. PiperOrigin-RevId: 509929075 --- src/google/protobuf/generated_message_reflection.cc | 7 ++++--- src/google/protobuf/wire_format.cc | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 41d2d9667e75..80a3eb5e914c 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -1339,8 +1339,8 @@ void Reflection::ClearField(Message* message, case FieldDescriptor::CPPTYPE_MESSAGE: if (schema_.HasBitIndex(field) == static_cast(-1)) { - // Some fields do not have hasbits and we need to set a message - // field to nullptr in order to indicate its un-presence. + // Proto3 does not have has-bits and we need to set a message field + // to nullptr in order to indicate its un-presence. if (message->GetArenaForAllocation() == nullptr) { delete *MutableRaw(message, field); } @@ -1554,7 +1554,8 @@ void CheckInOrder(const FieldDescriptor* field, uint32_t* last) { namespace internal { bool CreateUnknownEnumValues(const FieldDescriptor* field) { - return field->enum_type() == nullptr || !field->enum_type()->is_closed(); + bool open_enum = false; + return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || open_enum; } } // namespace internal using internal::CreateUnknownEnumValues; diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index e27678bf2ad3..27aa64fda93a 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -412,7 +412,7 @@ bool WireFormat::ParseAndMergeMessageSetField(uint32_t field_number, } static bool StrictUtf8Check(const FieldDescriptor* field) { - return field->requires_utf8_validation(); + return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3; } bool WireFormat::ParseAndMergeField( @@ -484,7 +484,8 @@ bool WireFormat::ParseAndMergeField( if (!WireFormatLite::ReadPrimitive( input, &value)) return false; - if (!field->enum_type()->is_closed()) { + if (message->GetDescriptor()->file()->syntax() == + FileDescriptor::SYNTAX_PROTO3) { message_reflection->AddEnumValue(message, field, value); } else { const EnumValueDescriptor* enum_value = @@ -868,7 +869,8 @@ const char* WireFormat::_InternalParseAndMergeField( auto rep_enum = reflection->MutableRepeatedFieldInternal(msg, field); bool open_enum = false; - if (!field->enum_type()->is_closed() || open_enum) { + if (field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || + open_enum) { ptr = internal::PackedEnumParser(rep_enum, ptr, ctx); } else { return ctx->ReadPackedVarint( From 9ced4cb36596bc69df063c36677583b2901aa4e0 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 15 Feb 2023 15:09:52 -0800 Subject: [PATCH 043/463] Implement a workaround for macOS/Bazel flakes. Bazel has a 2 minute timeout for their internal `xcrun` call, which can be exceeded on our github runners about 5% of the time. This leads to flakes and opaque errors, but is a one-time cost. Subsequent xcruns finish in seconds, so we can just do an initial call w/o a timeout before running Bazel. With this change our total flake rate drops from ~30% to nearly 0% for our full suite of tests See https://github.com/bazelbuild/bazel/issues/17437 for background. PiperOrigin-RevId: 509944178 --- .github/actions/bazel/action.yml | 23 +++++++++++++++++++ .../actions/internal/bazel-setup/action.yml | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml index 141527db0267..5fd3e9a90171 100644 --- a/.github/actions/bazel/action.yml +++ b/.github/actions/bazel/action.yml @@ -93,6 +93,29 @@ runs: shell: bash run: bazelisk version + # Bazel has multiple Xcode calls with hardcoded timeouts. Many of these + # end up timing out on our github runners, causing flakes on every mac + # build that invoked Bazel. To work around this, we manually inoke these + # calls before running Bazel to make sure they end up in Xcode's cache for + # quicker runs later. All of these calls are obtained from xcrun calls in + # https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl. + # See https://github.com/bazelbuild/bazel/issues/17437 for more details. + # TODO(b/269503614) Remove this once Bazel provides an official solution. + - name: Warm up Xcode + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + mkdir -p mac_bazel_workaround + bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS + XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation" + SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3" + COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3" + time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m + time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc + time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc + time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc + time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc + - name: Run Bash if: ${{ inputs.bash }} run: ${{ inputs.bash }} diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml index 20e724199fde..0ef5d5a2266a 100644 --- a/.github/actions/internal/bazel-setup/action.yml +++ b/.github/actions/internal/bazel-setup/action.yml @@ -42,7 +42,9 @@ runs: - name: Initialize MacOS-specific Bazel flags if: runner.os == 'macOS' shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV + run: | + echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV + echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV - name: Configure Bazel caching # Skip bazel cache for local act runs due to issue with credential files From 8d8527b77236a8df30ea99bd7061e32f78b85c86 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 15:18:06 -0800 Subject: [PATCH 044/463] Remove the exception for multiple extension ranges for TDP. It is unnecessary since the regular extension parser can handle unknown fields directly. PiperOrigin-RevId: 509946221 --- .../compiler/cpp/parse_function_generator.cc | 10 ++--- .../protobuf/generated_message_reflection.cc | 23 ++++++----- .../protobuf/generated_message_tctable_decl.h | 5 --- .../protobuf/generated_message_tctable_gen.cc | 3 +- .../protobuf/generated_message_tctable_impl.h | 17 ++++---- .../generated_message_tctable_lite_test.cc | 40 +++++++++---------- 6 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 7e93fcd6c2b3..791a7d5503bd 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -494,14 +494,10 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { } else { format("0, // no _has_bits_\n"); } - if (descriptor_->extension_range_count() == 1) { - format( - "PROTOBUF_FIELD_OFFSET($classname$, $extensions$),\n" - "$1$, $2$, // extension_range_{low,high}\n", - descriptor_->extension_range(0)->start, - descriptor_->extension_range(0)->end); + if (descriptor_->extension_range_count() != 0) { + format("PROTOBUF_FIELD_OFFSET($classname$, $extensions$),\n"); } else { - format("0, 0, 0, // no _extensions_\n"); + format("0, // no _extensions_\n"); } format("$1$, $2$, // max_field_number, fast_idx_mask\n", (ordered_fields_.empty() ? 0 : ordered_fields_.back()->number()), diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 80a3eb5e914c..efe5d7aae211 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -3092,9 +3092,9 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTableForMessageSet() // We use `operator new` here because the destruction will be done with // `operator delete` unconditionally. void* p = ::operator new(sizeof(Table)); - auto* full_table = ::new (p) Table{ - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, schema_.default_instance_, nullptr}, - {{{&internal::TcParser::ReflectionParseLoop, {}}}}}; + auto* full_table = ::new (p) + Table{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, schema_.default_instance_, nullptr}, + {{{&internal::TcParser::ReflectionParseLoop, {}}}}}; ABSL_DCHECK_EQ(static_cast(&full_table->header), static_cast(full_table)); return &full_table->header; @@ -3292,14 +3292,19 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { void* p = ::operator new(byte_size); auto* res = ::new (p) TcParseTableBase{ static_cast(schema_.HasHasbits() ? schema_.HasBitsOffset() : 0), - // extensions handled through reflection. - 0, 0, 0, + schema_.HasExtensionSet() + ? static_cast(schema_.GetExtensionSetOffset()) + : uint16_t{0}, static_cast(fields.empty() ? 0 : fields.back()->number()), - static_cast((fast_entries_count - 1) << 3), lookup_table_offset, - table_info.num_to_entry_table.skipmap32, field_entry_offset, + static_cast((fast_entries_count - 1) << 3), + lookup_table_offset, + table_info.num_to_entry_table.skipmap32, + field_entry_offset, static_cast(fields.size()), - static_cast(table_info.aux_entries.size()), aux_offset, - schema_.default_instance_, &internal::TcParser::ReflectionFallback}; + static_cast(table_info.aux_entries.size()), + aux_offset, + schema_.default_instance_, + &internal::TcParser::ReflectionFallback}; // Now copy the rest of the payloads PopulateTcParseFastEntries(table_info, res->fast_entry(0)); diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index 4ec6973d2332..33b094efff19 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -170,8 +170,6 @@ struct alignas(uint64_t) TcParseTableBase { // Common attributes for message layout: uint16_t has_bits_offset; uint16_t extension_offset; - uint32_t extension_range_low; - uint32_t extension_range_high; uint32_t max_field_number; uint8_t fast_idx_mask; uint16_t lookup_table_offset; @@ -194,7 +192,6 @@ struct alignas(uint64_t) TcParseTableBase { // compiled. constexpr TcParseTableBase( uint16_t has_bits_offset, uint16_t extension_offset, - uint32_t extension_range_low, uint32_t extension_range_high, uint32_t max_field_number, uint8_t fast_idx_mask, uint16_t lookup_table_offset, uint32_t skipmap32, uint32_t field_entries_offset, uint16_t num_field_entries, @@ -202,8 +199,6 @@ struct alignas(uint64_t) TcParseTableBase { const MessageLite* default_instance, TailCallParseFunc fallback) : has_bits_offset(has_bits_offset), extension_offset(extension_offset), - extension_range_low(extension_range_low), - extension_range_high(extension_range_high), max_field_number(max_field_number), fast_idx_mask(fast_idx_mask), lookup_table_offset(lookup_table_offset), diff --git a/src/google/protobuf/generated_message_tctable_gen.cc b/src/google/protobuf/generated_message_tctable_gen.cc index b829bb761ecd..a5791f92178a 100644 --- a/src/google/protobuf/generated_message_tctable_gen.cc +++ b/src/google/protobuf/generated_message_tctable_gen.cc @@ -824,8 +824,7 @@ TailCallTableInfo::TailCallTableInfo( // If there are no fallback fields, and at most one extension range, the // parser can use a generic fallback function. Otherwise, a message-specific // fallback routine is needed. - use_generated_fallback = - !fallback_fields.empty() || descriptor->extension_range_count() > 1; + use_generated_fallback = !fallback_fields.empty(); } } // namespace internal diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 84ee0e12841a..1adea2d9f91a 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -602,18 +602,21 @@ class PROTOBUF_EXPORT TcParser final { return ptr; } - uint32_t num = tag >> 3; - if (table->extension_range_low <= num && - num <= table->extension_range_high) { + if (table->extension_offset != 0) { + // We don't need to check the extension ranges. If it is not an extension + // it will be handled just like if it was an unknown extension: sent to + // the unknown field set. return RefAt(msg, table->extension_offset) .ParseField(tag, ptr, static_cast(table->default_instance), &msg->_internal_metadata_, ctx); + } else { + // Otherwise, we directly put it on the unknown field set. + return UnknownFieldParse( + tag, + msg->_internal_metadata_.mutable_unknown_fields(), + ptr, ctx); } - - return UnknownFieldParse( - tag, msg->_internal_metadata_.mutable_unknown_fields(), - ptr, ctx); } // Note: `inline` is needed on template function declarations below to avoid diff --git a/src/google/protobuf/generated_message_tctable_lite_test.cc b/src/google/protobuf/generated_message_tctable_lite_test.cc index 8aeacf57a28f..12aeced54459 100644 --- a/src/google/protobuf/generated_message_tctable_lite_test.cc +++ b/src/google/protobuf/generated_message_tctable_lite_test.cc @@ -94,7 +94,7 @@ TEST(FastVarints, NameHere) { const TcParseTable<0, 1, 0, 0, 2> parse_table = { { kHasBitsOffset, // - 0, 0, 0, // no _extensions_ + 0, // no _extensions_ 1, 0, // max_field_number, fast_idx_mask offsetof(decltype(parse_table), field_lookup_table), 0xFFFFFFFF - 1, // skipmap @@ -285,9 +285,9 @@ TEST(IsEntryForFieldNumTest, Matcher) { TcParseTable<0, 3, 0, 0, 2> table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 0, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 0, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF - 7, // 7 = fields 1, 2, and 3. offsetof(decltype(table), field_names), @@ -352,9 +352,9 @@ TEST_F(FindFieldEntryTest, SequentialFieldRange) { TcParseTable<0, 5, 0, 0, 8> table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 111, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 111, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF - (1 << 1) - (1 << 2) // fields 2, 3 - (1 << 3) - (1 << 4), // fields 4, 5 @@ -393,9 +393,9 @@ TEST_F(FindFieldEntryTest, SmallScanRange) { TcParseTable<0, 6, 0, 0, 8> table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 111, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 111, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF - (1<<0) - (1<<2) - (1<<3) - (1<<4) - (1<<6), // 1,3-5,7 offsetof(decltype(table), field_entries), @@ -439,9 +439,9 @@ TEST_F(FindFieldEntryTest, BinarySearchRange) { TcParseTable<0, 10, 0, 0, 8> table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 70, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 70, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF - (1<<0) - (1<<2) - (1<<3) - (1<<4) // 1, 3, 4, 5, 6 - (1<<5) - (1<<7) - (1<<8) - (1<<10) // 8, 9, 11, 12 @@ -485,9 +485,9 @@ TEST_F(FindFieldEntryTest, OutOfRange) { TcParseTable<0, 3, 0, 15, 2> table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 3, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 3, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF - (1<<0) - (1<<1) - (1<<2), // fields 1, 2, 3 offsetof(decltype(table), field_entries), @@ -535,9 +535,9 @@ TEST_F(FindFieldEntryTest, EmptyMessage) { TableType table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions - 0, // max_field_number - 0, // fast_idx_mask, + 0, 0, // has_bits_offset, extensions + 0, // max_field_number + 0, // fast_idx_mask, offsetof(decltype(table), field_lookup_table), 0xFFFFFFFF, // no fields offsetof(decltype(table), field_names), // no field_entries @@ -586,7 +586,7 @@ int32_t test_all_types_table_field_numbers[] = { const TcParseTable<5, 134, 5, 2176, 55> test_all_types_table = { // header: { - 0, 0, 0, 0, // has_bits_offset, extensions + 0, 0, // has_bits_offset, extensions 418, 248, // max_field_number, fast_idx_mask offsetof(decltype(test_all_types_table), field_lookup_table), 977895424, // skipmap for fields 1-15,18-19,21-22,24-25,27,31-32 From 1a90e67ff267eae2fb7cf94c85bb41956d4a7d59 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 15:20:10 -0800 Subject: [PATCH 045/463] [ObjC] Remove unused internal method. PiperOrigin-RevId: 509946709 --- objectivec/GPBUnknownFieldSet.m | 7 ------- objectivec/GPBUnknownFieldSet_PackagePrivate.h | 1 - objectivec/Tests/GPBUnknownFieldSetTest.m | 3 ++- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index dcd4b440a0c7..1dce73893405 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -289,13 +289,6 @@ - (void)mergeUnknownFields:(GPBUnknownFieldSet *)other { } } -- (void)mergeFromData:(NSData *)data { - GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; - [self mergeFromCodedInputStream:input]; - [input checkLastTagWas:0]; - [input release]; -} - - (void)mergeVarintField:(int32_t)number value:(int32_t)value { checkNumber(number); [[self mutableFieldForNumber:number create:YES] addVarint:value]; diff --git a/objectivec/GPBUnknownFieldSet_PackagePrivate.h b/objectivec/GPBUnknownFieldSet_PackagePrivate.h index e27127ad1d1e..b65bdaa1a83c 100644 --- a/objectivec/GPBUnknownFieldSet_PackagePrivate.h +++ b/objectivec/GPBUnknownFieldSet_PackagePrivate.h @@ -50,7 +50,6 @@ - (void)mergeUnknownFields:(GPBUnknownFieldSet *)other; - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input; -- (void)mergeFromData:(NSData *)data; - (void)mergeVarintField:(int32_t)number value:(int32_t)value; - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input; diff --git a/objectivec/Tests/GPBUnknownFieldSetTest.m b/objectivec/Tests/GPBUnknownFieldSetTest.m index b98f52ef0877..dbb6eb488f69 100644 --- a/objectivec/Tests/GPBUnknownFieldSetTest.m +++ b/objectivec/Tests/GPBUnknownFieldSetTest.m @@ -381,7 +381,8 @@ - (void)testLargeVarint { NSData* data = [fields data]; GPBUnknownFieldSet* parsed = [[[GPBUnknownFieldSet alloc] init] autorelease]; - [parsed mergeFromData:data]; + GPBCodedInputStream* input = [[[GPBCodedInputStream alloc] initWithData:data] autorelease]; + [parsed mergeFromCodedInputStream:input]; GPBUnknownField* field2 = [parsed getField:1]; XCTAssertEqual(field2.varintList.count, (NSUInteger)1); XCTAssertEqual(0x7FFFFFFFFFFFFFFFULL, [field2.varintList valueAtIndex:0]); From c4ed9483146ac633a9f725f307f9ac60074d0f4d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 15 Feb 2023 18:37:48 -0800 Subject: [PATCH 046/463] Fix lint warnings for protobuf GeneratedMessage.java PiperOrigin-RevId: 509989425 --- .../google/protobuf/GeneratedMessageV3.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index ab9581acda0a..b25011e0d9ff 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -423,6 +423,26 @@ protected static IntList mutableCopy(IntList list) { return makeMutableCopy(list); } + // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. + protected static LongList mutableCopy(LongList list) { + return makeMutableCopy(list); + } + + // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. + protected static FloatList mutableCopy(FloatList list) { + return makeMutableCopy(list); + } + + // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. + protected static DoubleList mutableCopy(DoubleList list) { + return makeMutableCopy(list); + } + + // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. + protected static BooleanList mutableCopy(BooleanList list) { + return makeMutableCopy(list); + } + protected static LongList emptyLongList() { return LongArrayList.emptyList(); } @@ -432,11 +452,6 @@ protected static LongList newLongList() { return new LongArrayList(); } - // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. - protected static LongList mutableCopy(LongList list) { - return makeMutableCopy(list); - } - protected static FloatList emptyFloatList() { return FloatArrayList.emptyList(); } @@ -446,11 +461,6 @@ protected static FloatList newFloatList() { return new FloatArrayList(); } - // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. - protected static FloatList mutableCopy(FloatList list) { - return makeMutableCopy(list); - } - protected static DoubleList emptyDoubleList() { return DoubleArrayList.emptyList(); } @@ -460,11 +470,6 @@ protected static DoubleList newDoubleList() { return new DoubleArrayList(); } - // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. - protected static DoubleList mutableCopy(DoubleList list) { - return makeMutableCopy(list); - } - protected static BooleanList emptyBooleanList() { return BooleanArrayList.emptyList(); } @@ -474,11 +479,6 @@ protected static BooleanList newBooleanList() { return new BooleanArrayList(); } - // TODO(b/258340024): Redundant with makeMutableCopy(). Remove. - protected static BooleanList mutableCopy(BooleanList list) { - return makeMutableCopy(list); - } - @SuppressWarnings("unchecked") // Guaranteed by proto runtime. protected static > ListT makeMutableCopy(ListT list) { int size = list.size(); From 2fb35fcf56ec8502538256334d0207ef1555b560 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 16 Feb 2023 07:19:15 -0800 Subject: [PATCH 047/463] Automated rollback of commit 0c013ec412468e21ea68b234935e7490eb28f8ce. PiperOrigin-RevId: 510136389 --- objectivec/GPBMessage.m | 40 +++++++++++++------ objectivec/GPBMessage_PackagePrivate.h | 5 +++ .../Tests/GPBMessageTests+Serialization.m | 2 +- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 84953476d146..cb8d70a47573 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -2000,6 +2000,19 @@ - (void)mergeFromData:(NSData *)data extensionRegistry:(id [input release]; } +#pragma mark - mergeDelimitedFrom + +- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input + extensionRegistry:(id)extensionRegistry { + GPBCodedInputStreamState *state = &input->state_; + if (GPBCodedInputStreamIsAtEnd(state)) { + return; + } + NSData *data = GPBCodedInputStreamReadRetainedBytesNoCopy(state); + [self mergeFromData:data extensionRegistry:extensionRegistry]; + [data release]; +} + #pragma mark - Parse From Data Support + (instancetype)parseFromData:(NSData *)data error:(NSError **)errorPtr { @@ -2026,26 +2039,27 @@ + (instancetype)parseFromCodedInputStream:(GPBCodedInputStream *)input + (instancetype)parseDelimitedFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { - GPBMessage *result = nil; + GPBMessage *message = [[[self alloc] init] autorelease]; @try { - uint64_t size = GPBCodedInputStreamReadUInt64(&input->state_); - size_t oldLimit = [input pushLimit:size]; - result = [self parseFromCodedInputStream:input - extensionRegistry:extensionRegistry - error:errorPtr]; - if (result) { - [input popLimit:oldLimit]; - if (errorPtr) { - *errorPtr = nil; - } + [message mergeDelimitedFromCodedInputStream:input extensionRegistry:extensionRegistry]; + if (errorPtr) { + *errorPtr = nil; } } @catch (NSException *exception) { - result = nil; + message = nil; if (errorPtr) { *errorPtr = ErrorFromException(exception); } } - return result; +#ifdef DEBUG + if (message && !message.initialized) { + message = nil; + if (errorPtr) { + *errorPtr = MessageError(GPBMessageErrorCodeMissingRequiredField, nil); + } + } +#endif + return message; } #pragma mark - Unknown Field Support diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 1b3a99332df8..70b47a59bc88 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -80,6 +80,11 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry; +// Parses the next delimited message of this type from the input and merges it +// with this message. +- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input + extensionRegistry:(id)extensionRegistry; + - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data; @end diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index 68b62ef7acf8..5a7dc16c84df 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -1276,7 +1276,7 @@ - (void)testParseDelimitedDataOver2GB { XCTAssertNil(result); XCTAssertNotNil(error); XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); - XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSubsectionLimit); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSize); } #ifdef DEBUG From ff91b25494e353e61d6ad37aafb70f45c328b1ce Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 16 Feb 2023 09:02:19 -0800 Subject: [PATCH 048/463] Use generated Conformance proto for Bazel targets. PiperOrigin-RevId: 510158778 --- csharp/BUILD.bazel | 2 ++ csharp/src/Google.Protobuf.Conformance/BUILD.bazel | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/csharp/BUILD.bazel b/csharp/BUILD.bazel index 23fbe411f9eb..2af7d067af98 100644 --- a/csharp/BUILD.bazel +++ b/csharp/BUILD.bazel @@ -65,9 +65,11 @@ inline_sh_test( srcs = [ ":srcs", "src/Google.Protobuf.sln", + "//conformance:conformance_csharp_proto", "//csharp/src/Google.Protobuf.Conformance:srcs", ], cmd = """ + cp $(rootpath //conformance:conformance_csharp_proto) `dirname $(location src/Google.Protobuf.sln)`/Google.Protobuf.Conformance/ pushd `dirname $(location src/Google.Protobuf.sln)`/.. dotnet restore src/Google.Protobuf.sln dotnet build -c Release src/Google.Protobuf.sln diff --git a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel index 358c44c10888..0b5e2c8f83de 100644 --- a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel +++ b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel @@ -1,3 +1,4 @@ +load("//:protobuf.bzl", "internal_csharp_proto_library") load("//build_defs:internal_shell.bzl", "inline_sh_binary") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") @@ -37,7 +38,6 @@ CONFORMANCE_DEPS = [ filegroup( name = "srcs", srcs = [ - "Conformance.pb.cs", "Program.cs", "Google.Protobuf.Conformance.csproj", ], @@ -88,7 +88,6 @@ pkg_files( srcs = [ "BUILD.bazel", "Google.Protobuf.Conformance.csproj", - "Conformance.pb.cs", "Program.cs", ], strip_prefix = strip_prefix.from_root(""), From c4481302f031a83baa7678650f1e29e8d7252cbd Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 16 Feb 2023 09:30:31 -0800 Subject: [PATCH 049/463] Pull any new changes before pushing staleness updates. This fixes some relatively harmless flakiness in our post-submit. If two commits are made within ~5 minutes of each other, the `git push` command will fail because the git workspace isn't up to date. PiperOrigin-RevId: 510165977 --- push_auto_update.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/push_auto_update.sh b/push_auto_update.sh index 0d0e4371b0b2..9972da6acf51 100755 --- a/push_auto_update.sh +++ b/push_auto_update.sh @@ -39,6 +39,7 @@ else commit_message="Auto-generate files" fi +git pull --rebase git add -A git diff --staged --quiet || git commit -am "$commit_message" git push From ee8bbc3c62b57252a225d23c952f277da32ec04b Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 16 Feb 2023 19:26:24 +0000 Subject: [PATCH 050/463] Restore codeowners and version.json --- .github/CODEOWNERS | 37 +++++++++++++++++++++++++++++++++++++ version.json | 24 ++++++++++++------------ 2 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..f3206a4f6256 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,37 @@ +/src/ @protocolbuffers/protobuf-compiler + +/src/google/protobuf/compiler/cpp @protocolbuffers/protobuf-cpp + +/csharp/ @protocolbuffers/protobuf-csharp +/src/google/protobuf/compiler/csharp/ @protocolbuffers/protobuf-csharp + +/docs/ @protocolbuffers/protobuf-docs +/examples/ @protocolbuffers/protobuf-docs + +/java/ @protocolbuffers/protobuf-java +/src/google/protobuf/compiler/java/ @protocolbuffers/protobuf-java + +/java/kotlin-lite/ @protocolbuffers/protobuf-kotlin +/java/kotlin/ @protocolbuffers/protobuf-kotlin + +/objectivec/ @protocolbuffers/protobuf-objc +/src/google/protobuf/compiler/objectivec/ @protocolbuffers/protobuf-objc + +/php/ @protocolbuffers/protobuf-php +/src/google/protobuf/compiler/php/ @protocolbuffers/protobuf-php + +/python/ @protocolbuffers/protobuf-python +/src/google/protobuf/compiler/python/ @protocolbuffers/protobuf-python + +/ruby/ @protocolbuffers/protobuf-ruby +/src/google/protobuf/compiler/ruby/ @protocolbuffers/protobuf-ruby + +/build_defs/ @protocolbuffers/protobuf-btr +/cmake/ @protocolbuffers/protobuf-btr +/pkg/ @protocolbuffers/protobuf-btr +/toolchain/ @protocolbuffers/protobuf-btr +/conformance/ @protocolbuffers/protobuf-btr +/kokoro/ @protocolbuffers/protobuf-btr +/third_party/ @protocolbuffers/protobuf-btr +*.bazel @protocolbuffers/protobuf-btr +/.github/ @protocolbuffers/protobuf-btr diff --git a/version.json b/version.json index bc12f0e68185..695db4b9e73b 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { - "22.x": { - "protoc_version": "22.1-dev", + "main": { + "protoc_version": "22-dev", "lts": false, - "date": "2023-02-16", + "date": "2022-07-21", "languages": { - "cpp": "4.22.1-dev", - "csharp": "3.22.1-dev", - "java": "3.22.1-dev", - "javascript": "3.22.1-dev", - "objectivec": "3.22.1-dev", - "php": "3.22.1-dev", - "python": "4.22.1-dev", - "ruby": "3.22.1-dev" + "cpp": "4.22-dev", + "csharp": "3.22-dev", + "java": "3.22-dev", + "javascript": "3.22-dev", + "objectivec": "3.22-dev", + "php": "3.22-dev", + "python": "4.22-dev", + "ruby": "3.22-dev" } } -} \ No newline at end of file +} From eaf4e268fa7c08fb6c3fc0050b2bfd7466caad04 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 16 Feb 2023 11:25:33 -0800 Subject: [PATCH 051/463] Discard remaining tag but check against non canonical encoding. PiperOrigin-RevId: 510197783 --- src/google/protobuf/message_unittest.inc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index c41d8b3d0e78..4c8ffdcb500d 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -324,6 +324,13 @@ TEST(MESSAGE_TEST_NAME, ExplicitLazyExceedRecursionLimit) { EXPECT_NE(parsed.lazy_child().child().payload().optional_int32(), -1); } +TEST(MESSAGE_TEST_NAME, ParseFailNonCanonicalZeroTag) { + const char encoded[] = {"\n\x3\x80\0\0"}; + UNITTEST::NestedTestAllTypes parsed; + EXPECT_FALSE(parsed.ParsePartialFromString( + absl::string_view{encoded, sizeof(encoded) - 1})); +} + TEST(MESSAGE_TEST_NAME, NestedExplicitLazyExceedRecursionLimit) { UNITTEST::NestedTestAllTypes original, parsed; // Build proto with recursion depth of 5, with nested annotated LazyField. From 911847efc45abc90fea849ece545b2cda63e858c Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 16 Feb 2023 11:42:17 -0800 Subject: [PATCH 052/463] Cache our ccache download to avoid network flakes from choco. Chocolatey doesn't support any caching, so we're moving back to downloading the exe directly. This issue is responsible for a 5-10% flake rate in our windows cmake builds PiperOrigin-RevId: 510202223 --- .../internal/ccache-setup-windows/action.yml | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml index a42e7bf8a8ed..7b9f9925b35d 100644 --- a/.github/actions/internal/ccache-setup-windows/action.yml +++ b/.github/actions/internal/ccache-setup-windows/action.yml @@ -16,9 +16,41 @@ runs: arch: x64 vsversion: '2019' - - name: Install ccache + - name: Setup ccache path shell: bash - run: choco install ccache --version=4.7.4 + run: | + echo "CCACHE_EXE_PATH=$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_ENV + echo "$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_PATH + + - name: Add ccache to Powershell path + shell: pwsh + run: echo "%LOCALAPPDATA%\ccache-${{ inputs.ccache-version }}-windows-x86_64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Setup caching of ccache download + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} + id: ccache-cache + uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 + with: + path: ${{ env.CCACHE_EXE_PATH }} + key: ccache-exe-${{ inputs.ccache-version }} + + - name: Restore ccache download + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + id: ccache-restore + uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 + with: + path: ${{ env.CCACHE_EXE_PATH }} + key: ccache-exe-${{ inputs.ccache-version }} + + - name: Download ccache + shell: bash + if: ${{ steps.ccache-cache.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'}} + run: | + cd $LOCALAPPDATA + curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip + unzip ccache.zip + rm ccache.zip + ccache --version - name: Configure ccache environment variables shell: pwsh From 7f6b36ce0cea86d60ba70c51723870397d88a246 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 16 Feb 2023 20:25:11 +0000 Subject: [PATCH 053/463] Revert UPB dep update --- build_defs/upb.patch | 20 +- php/ext/google/protobuf/php-upb.c | 311 +++++++-------------- php/ext/google/protobuf/php-upb.h | 387 +++++++++++++------------- protobuf_deps.bzl | 4 +- ruby/ext/google/protobuf_c/ruby-upb.c | 311 +++++++-------------- ruby/ext/google/protobuf_c/ruby-upb.h | 387 +++++++++++++------------- 6 files changed, 619 insertions(+), 801 deletions(-) diff --git a/build_defs/upb.patch b/build_defs/upb.patch index 809fb7054966..ec1f28b50040 100644 --- a/build_defs/upb.patch +++ b/build_defs/upb.patch @@ -9,4 +9,22 @@ # end:github_only def _upbc(stage): - \ No newline at end of file + +--- cmake/build_defs.bzl ++++ cmake/build_defs.bzl +@@ -25,7 +25,7 @@ + + """Bazel support functions related to CMake support.""" + +-def staleness_test(name, outs, generated_pattern, target_files = None, **kwargs): ++def staleness_test(name, outs, generated_pattern, target_files = None, tags = [], **kwargs): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the +@@ -72,5 +72,6 @@ def staleness_test(name, outs, generated_pattern, target_files = None, **kwargs) + deps = [ + Label("//cmake:staleness_test_lib"), + ], ++ tags = ["staleness_test"] + tags, + **kwargs + ) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 2dac3ad0cb94..ef9572417e46 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -641,12 +641,14 @@ static int (*const compar[kUpb_FieldType_SizeOf])(const void*, const void*) = { [kUpb_FieldType_Bytes] = _upb_mapsorter_cmpstr, }; -static bool _upb_mapsorter_resize(_upb_mapsorter* s, _upb_sortedmap* sorted, - int size) { +bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, + const upb_Map* map, _upb_sortedmap* sorted) { + int map_size = _upb_Map_Size(map); sorted->start = s->size; sorted->pos = sorted->start; - sorted->end = sorted->start + size; + sorted->end = sorted->start + map_size; + // Grow s->entries if necessary. if (sorted->end > s->cap) { s->cap = upb_Log2CeilingSize(sorted->end); s->entries = realloc(s->entries, s->cap * sizeof(*s->entries)); @@ -654,17 +656,9 @@ static bool _upb_mapsorter_resize(_upb_mapsorter* s, _upb_sortedmap* sorted, } s->size = sorted->end; - return true; -} - -bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, - const upb_Map* map, _upb_sortedmap* sorted) { - int map_size = _upb_Map_Size(map); - - if (!_upb_mapsorter_resize(s, sorted, map_size)) return false; // Copy non-empty entries from the table to s->entries. - const void** dst = &s->entries[sorted->start]; + upb_tabent const** dst = &s->entries[sorted->start]; const upb_tabent* src = map->table.t.entries; const upb_tabent* end = src + upb_table_size(&map->table.t); for (; src < end; src++) { @@ -680,29 +674,6 @@ bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, compar[key_type]); return true; } - -static int _upb_mapsorter_cmpext(const void* _a, const void* _b) { - const upb_Message_Extension* const* a = _a; - const upb_Message_Extension* const* b = _b; - uint32_t a_num = (*a)->ext->field.number; - uint32_t b_num = (*b)->ext->field.number; - assert(a_num != b_num); - return a_num < b_num ? -1 : 1; -} - -bool _upb_mapsorter_pushexts(_upb_mapsorter* s, - const upb_Message_Extension* exts, size_t count, - _upb_sortedmap* sorted) { - if (!_upb_mapsorter_resize(s, sorted, count)) return false; - - for (size_t i = 0; i < count; i++) { - s->entries[sorted->start + i] = &exts[i]; - } - - qsort(&s->entries[sorted->start], count, sizeof(*s->entries), - _upb_mapsorter_cmpext); - return true; -} /* This file was generated by upbc (the upb compiler) from the input * file: * @@ -2254,7 +2225,7 @@ char* upb_strdup2(const char* s, size_t len, upb_Arena* a) { n = len + 1; p = upb_Arena_Malloc(a, n); if (p) { - if (len != 0) memcpy(p, s, len); + memcpy(p, s, len); p[len] = 0; } return p; @@ -7590,27 +7561,9 @@ void _upb_DefBuilder_OomErr(upb_DefBuilder* ctx) { _upb_DefBuilder_FailJmp(ctx); } -// Verify a relative identifier string. The loop is branchless for speed. -static void _upb_DefBuilder_CheckIdentNotFull(upb_DefBuilder* ctx, - upb_StringView name) { - bool good = name.size > 0; - - for (size_t i = 0; i < name.size; i++) { - const char c = name.data[i]; - const char d = c | 0x20; // force lowercase - const bool is_alpha = (('a' <= d) & (d <= 'z')) | (c == '_'); - const bool is_numer = ('0' <= c) & (c <= '9') & (i != 0); - - good &= is_alpha | is_numer; - } - - if (!good) _upb_DefBuilder_CheckIdentSlow(ctx, name, false); -} - const char* _upb_DefBuilder_MakeFullName(upb_DefBuilder* ctx, const char* prefix, upb_StringView name) { - _upb_DefBuilder_CheckIdentNotFull(ctx, name); if (prefix) { // ret = prefix + '.' + name; size_t n = strlen(prefix); @@ -7726,7 +7679,7 @@ static bool TryGetChar(const char** src, const char* end, char* ch) { return true; } -static int TryGetHexDigit(const char** src, const char* end) { +static char TryGetHexDigit(const char** src, const char* end) { char ch; if (!TryGetChar(src, end, &ch)) return -1; if ('0' <= ch && ch <= '9') { @@ -7743,10 +7696,10 @@ static int TryGetHexDigit(const char** src, const char* end) { static char upb_DefBuilder_ParseHexEscape(upb_DefBuilder* ctx, const upb_FieldDef* f, const char** src, const char* end) { - int hex_digit = TryGetHexDigit(src, end); + char hex_digit = TryGetHexDigit(src, end); if (hex_digit < 0) { _upb_DefBuilder_Errf( - ctx, "\\x must be followed by at least one hex digit (field='%s')", + ctx, "\\x cannot be followed by non-hex digit in field '%s' default", upb_FieldDef_FullName(f)); return 0; } @@ -7922,7 +7875,7 @@ upb_DefPool* upb_DefPool_New(void) { } bool _upb_DefPool_InsertExt(upb_DefPool* s, const upb_MiniTableExtension* ext, - const upb_FieldDef* f) { + upb_FieldDef* f) { return upb_inttable_insert(&s->exts, (uintptr_t)ext, upb_value_constptr(f), s->arena); } @@ -8154,6 +8107,12 @@ static const upb_FileDef* _upb_DefPool_AddFile( const upb_MiniTableFile* layout, upb_Status* status) { const upb_StringView name = UPB_DESC(FileDescriptorProto_name)(file_proto); + if (name.size == 0) { + upb_Status_SetErrorFormat(status, + "missing name in google_protobuf_FileDescriptorProto"); + return NULL; + } + // Determine whether we already know about this file. { upb_value v; @@ -8545,6 +8504,7 @@ static void create_enumdef(upb_DefBuilder* ctx, const char* prefix, e->file = _upb_DefBuilder_File(ctx); name = UPB_DESC(EnumDescriptorProto_name)(enum_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); e->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); _upb_DefBuilder_Add(ctx, e->full_name, @@ -9354,14 +9314,7 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, } const upb_StringView name = UPB_DESC(FieldDescriptorProto_name)(field_proto); - - f->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); - f->label_ = (int)UPB_DESC(FieldDescriptorProto_label)(field_proto); - f->number_ = UPB_DESC(FieldDescriptorProto_number)(field_proto); - f->is_proto3_optional = - UPB_DESC(FieldDescriptorProto_proto3_optional)(field_proto); - f->msgdef = m; - f->scope.oneof = NULL; + _upb_DefBuilder_CheckIdentNotFull(ctx, name); f->has_json_name = UPB_DESC(FieldDescriptorProto_has_json_name)(field_proto); if (f->has_json_name) { @@ -9373,6 +9326,14 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, } if (!f->json_name) _upb_DefBuilder_OomErr(ctx); + f->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); + f->label_ = (int)UPB_DESC(FieldDescriptorProto_label)(field_proto); + f->number_ = UPB_DESC(FieldDescriptorProto_number)(field_proto); + f->is_proto3_optional = + UPB_DESC(FieldDescriptorProto_proto3_optional)(field_proto); + f->msgdef = m; + f->scope.oneof = NULL; + const bool has_type = UPB_DESC(FieldDescriptorProto_has_type)(field_proto); const bool has_type_name = UPB_DESC(FieldDescriptorProto_has_type_name)(field_proto); @@ -9502,24 +9463,19 @@ static void _upb_FieldDef_CreateNotExt(upb_DefBuilder* ctx, const char* prefix, } _upb_MessageDef_InsertField(ctx, m, f); -} -upb_FieldDef* _upb_Extensions_New( - upb_DefBuilder* ctx, int n, - const UPB_DESC(FieldDescriptorProto) * const* protos, const char* prefix, - upb_MessageDef* m) { - _upb_DefType_CheckPadding(sizeof(upb_FieldDef)); - upb_FieldDef* defs = - (upb_FieldDef*)_upb_DefBuilder_Alloc(ctx, sizeof(upb_FieldDef) * n); + if (!ctx->layout) return; - for (int i = 0; i < n; i++) { - upb_FieldDef* f = &defs[i]; - - _upb_FieldDef_CreateExt(ctx, prefix, protos[i], m, f); - f->index_ = i; + const upb_MiniTable* mt = upb_MessageDef_MiniTable(m); + const upb_MiniTableField* fields = mt->fields; + for (int i = 0; i < mt->field_count; i++) { + if (fields[i].number == f->number_) { + f->layout_index = i; + return; + } } - return defs; + UPB_ASSERT(false); // It should be impossible to reach this point. } upb_FieldDef* _upb_FieldDefs_New( @@ -9530,23 +9486,28 @@ upb_FieldDef* _upb_FieldDefs_New( upb_FieldDef* defs = (upb_FieldDef*)_upb_DefBuilder_Alloc(ctx, sizeof(upb_FieldDef) * n); - uint32_t previous = 0; - for (int i = 0; i < n; i++) { - upb_FieldDef* f = &defs[i]; - - _upb_FieldDef_CreateNotExt(ctx, prefix, protos[i], m, f); - f->index_ = i; - if (!ctx->layout) { - // Speculate that the def fields are sorted. We will always sort the - // MiniTable fields, so if defs are sorted then indices will match. - // - // If this is incorrect, we will overwrite later. - f->layout_index = i; + // If we are creating extensions then is_sorted will be NULL. + // If we are not creating extensions then is_sorted will be non-NULL. + if (is_sorted) { + uint32_t previous = 0; + for (int i = 0; i < n; i++) { + upb_FieldDef* f = &defs[i]; + + _upb_FieldDef_CreateNotExt(ctx, prefix, protos[i], m, f); + f->index_ = i; + if (!ctx->layout) f->layout_index = i; + + const uint32_t current = f->number_; + if (previous > current) *is_sorted = false; + previous = current; } + } else { + for (int i = 0; i < n; i++) { + upb_FieldDef* f = &defs[i]; - const uint32_t current = f->number_; - if (previous > current) *is_sorted = false; - previous = current; + _upb_FieldDef_CreateExt(ctx, prefix, protos[i], m, f); + f->index_ = i; + } } return defs; @@ -9602,9 +9563,6 @@ static int _upb_FieldDef_Compare(const void* p1, const void* p2) { return (v1 < v2) ? -1 : (v1 > v2); } -// _upb_FieldDefs_Sorted() is mostly a pure function of its inputs, but has one -// critical side effect that we depend on: it sets layout_index appropriately -// for non-sorted lists of fields. const upb_FieldDef** _upb_FieldDefs_Sorted(const upb_FieldDef* f, int n, upb_Arena* a) { // TODO(salo): Replace this arena alloc with a persistent scratch buffer. @@ -9662,10 +9620,7 @@ static void resolve_extension(upb_DefBuilder* ctx, const char* prefix, "field number %u in extension %s has no extension range in message %s", (unsigned)f->number_, f->full_name, upb_MessageDef_FullName(m)); } -} -void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, - const upb_FieldDef* f) { const upb_MiniTableExtension* ext = _upb_FieldDef_ExtensionMiniTable(f); if (ctx->layout) { @@ -9684,8 +9639,8 @@ void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, sub.subenum = _upb_EnumDef_MiniTable(f->sub.enumdef); } bool ok2 = upb_MiniTableExtension_Build(desc.data, desc.size, mut_ext, - upb_MessageDef_MiniTable(f->msgdef), - sub, ctx->status); + upb_MessageDef_MiniTable(m), sub, + ctx->status); if (!ok2) _upb_DefBuilder_Errf(ctx, "Could not build extension mini table"); } @@ -9742,7 +9697,6 @@ struct upb_FileDef { const UPB_DESC(FileOptions) * opts; const char* name; const char* package; - const char* edition; const upb_FileDef** deps; const int32_t* public_deps; @@ -9779,10 +9733,6 @@ const char* upb_FileDef_Package(const upb_FileDef* f) { return f->package ? f->package : ""; } -const char* upb_FileDef_Edition(const upb_FileDef* f) { - return f->edition ? f->edition : ""; -} - const char* _upb_FileDef_RawPackage(const upb_FileDef* f) { return f->package; } upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f) { return f->syntax; } @@ -9930,14 +9880,13 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, } } - upb_StringView name = UPB_DESC(FileDescriptorProto_name)(file_proto); - file->name = strviewdup(ctx, name); - if (strlen(file->name) != name.size) { - _upb_DefBuilder_Errf(ctx, "File name contained embedded NULL"); + if (!UPB_DESC(FileDescriptorProto_has_name)(file_proto)) { + _upb_DefBuilder_Errf(ctx, "File has no name"); } - upb_StringView package = UPB_DESC(FileDescriptorProto_package)(file_proto); + file->name = strviewdup(ctx, UPB_DESC(FileDescriptorProto_name)(file_proto)); + upb_StringView package = UPB_DESC(FileDescriptorProto_package)(file_proto); if (package.size) { _upb_DefBuilder_CheckIdentFull(ctx, package); file->package = strviewdup(ctx, package); @@ -9945,18 +9894,6 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, file->package = NULL; } - upb_StringView edition = UPB_DESC(FileDescriptorProto_edition)(file_proto); - - if (edition.size == 0) { - file->edition = NULL; - } else { - // TODO(b/267770604): How should we validate this? - file->edition = strviewdup(ctx, edition); - if (strlen(file->edition) != edition.size) { - _upb_DefBuilder_Errf(ctx, "Edition name contained embedded NULL"); - } - } - if (UPB_DESC(FileDescriptorProto_has_syntax)(file_proto)) { upb_StringView syntax = UPB_DESC(FileDescriptorProto_syntax)(file_proto); @@ -10025,7 +9962,8 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, // Create extensions. exts = UPB_DESC(FileDescriptorProto_extension)(file_proto, &n); file->top_lvl_ext_count = n; - file->top_lvl_exts = _upb_Extensions_New(ctx, n, exts, file->package, NULL); + file->top_lvl_exts = + _upb_FieldDefs_New(ctx, n, exts, file->package, NULL, NULL); // Create messages. msgs = UPB_DESC(FileDescriptorProto_message_type)(file_proto, &n); @@ -10049,19 +9987,11 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, _upb_FieldDef_Resolve(ctx, file->package, f); } - for (int i = 0; i < file->top_lvl_msg_count; i++) { - upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); - _upb_MessageDef_CreateMiniTable(ctx, (upb_MessageDef*)m); - } - - for (int i = 0; i < file->top_lvl_ext_count; i++) { - upb_FieldDef* f = (upb_FieldDef*)upb_FileDef_TopLevelExtension(file, i); - _upb_FieldDef_BuildMiniTableExtension(ctx, f); - } - - for (int i = 0; i < file->top_lvl_msg_count; i++) { - upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); - _upb_MessageDef_LinkMiniTable(ctx, m); + if (!ctx->layout) { + for (int i = 0; i < file->top_lvl_msg_count; i++) { + upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); + _upb_MessageDef_LinkMiniTable(ctx, m); + } } if (file->ext_count) { @@ -10574,8 +10504,6 @@ bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m) { static upb_MiniTable* _upb_MessageDef_MakeMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m) { upb_StringView desc; - // Note: this will assign layout_index for fields, so upb_FieldDef_MiniTable() - // is safe to call only after this call. bool ok = upb_MessageDef_MiniDescriptorEncode(m, ctx->tmp_arena, &desc); if (!ok) _upb_DefBuilder_OomErr(ctx); @@ -10595,6 +10523,23 @@ void _upb_MessageDef_Resolve(upb_DefBuilder* ctx, upb_MessageDef* m) { _upb_FieldDef_Resolve(ctx, m->full_name, f); } + if (!ctx->layout) { + m->layout = _upb_MessageDef_MakeMiniTable(ctx, m); + if (!m->layout) _upb_DefBuilder_OomErr(ctx); + } + +#ifndef NDEBUG + for (int i = 0; i < m->field_count; i++) { + const upb_FieldDef* f = upb_MessageDef_Field(m, i); + const int layout_index = _upb_FieldDef_LayoutIndex(f); + UPB_ASSERT(layout_index < m->layout->field_count); + const upb_MiniTableField* mt_f = &m->layout->fields[layout_index]; + UPB_ASSERT(upb_FieldDef_Type(f) == upb_MiniTableField_Type(mt_f)); + UPB_ASSERT(upb_FieldDef_HasPresence(f) == + upb_MiniTableField_HasPresence(mt_f)); + } +#endif + m->in_message_set = false; for (int i = 0; i < upb_MessageDef_NestedExtensionCount(m); i++) { upb_FieldDef* ext = (upb_FieldDef*)upb_MessageDef_NestedExtension(m, i); @@ -10657,39 +10602,8 @@ void _upb_MessageDef_InsertField(upb_DefBuilder* ctx, upb_MessageDef* m, if (!ok) _upb_DefBuilder_OomErr(ctx); } -void _upb_MessageDef_CreateMiniTable(upb_DefBuilder* ctx, upb_MessageDef* m) { - if (ctx->layout == NULL) { - m->layout = _upb_MessageDef_MakeMiniTable(ctx, m); - } else { - UPB_ASSERT(ctx->msg_count < ctx->layout->msg_count); - m->layout = ctx->layout->msgs[ctx->msg_count++]; - UPB_ASSERT(m->field_count == m->layout->field_count); - - // We don't need the result of this call, but it will assign layout_index - // for all the fields in O(n lg n) time. - _upb_FieldDefs_Sorted(m->fields, m->field_count, ctx->tmp_arena); - } - - for (int i = 0; i < m->nested_msg_count; i++) { - upb_MessageDef* nested = - (upb_MessageDef*)upb_MessageDef_NestedMessage(m, i); - _upb_MessageDef_CreateMiniTable(ctx, nested); - } -} - void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m) { - for (int i = 0; i < upb_MessageDef_NestedExtensionCount(m); i++) { - const upb_FieldDef* ext = upb_MessageDef_NestedExtension(m, i); - _upb_FieldDef_BuildMiniTableExtension(ctx, ext); - } - - for (int i = 0; i < m->nested_msg_count; i++) { - _upb_MessageDef_LinkMiniTable(ctx, upb_MessageDef_NestedMessage(m, i)); - } - - if (ctx->layout) return; - for (int i = 0; i < m->field_count; i++) { const upb_FieldDef* f = upb_MessageDef_Field(m, i); const upb_MessageDef* sub_m = upb_FieldDef_MessageSubDef(f); @@ -10717,17 +10631,9 @@ void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, } } -#ifndef NDEBUG - for (int i = 0; i < m->field_count; i++) { - const upb_FieldDef* f = upb_MessageDef_Field(m, i); - const int layout_index = _upb_FieldDef_LayoutIndex(f); - UPB_ASSERT(layout_index < m->layout->field_count); - const upb_MiniTableField* mt_f = &m->layout->fields[layout_index]; - UPB_ASSERT(upb_FieldDef_Type(f) == upb_MiniTableField_Type(mt_f)); - UPB_ASSERT(upb_FieldDef_HasPresence(f) == - upb_MiniTableField_HasPresence(mt_f)); + for (int i = 0; i < m->nested_msg_count; i++) { + _upb_MessageDef_LinkMiniTable(ctx, upb_MessageDef_NestedMessage(m, i)); } -#endif } static uint64_t _upb_MessageDef_Modifiers(const upb_MessageDef* m) { @@ -10860,6 +10766,7 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, m->is_sorted = true; name = UPB_DESC(DescriptorProto_name)(msg_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); m->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); _upb_DefBuilder_Add(ctx, m->full_name, _upb_DefType_Pack(m, UPB_DEFTYPE_MSG)); @@ -10878,6 +10785,17 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, ok = upb_strtable_init(&m->ntof, n_oneof + n_field, ctx->arena); if (!ok) _upb_DefBuilder_OomErr(ctx); + if (ctx->layout) { + /* create_fielddef() below depends on this being set. */ + UPB_ASSERT(ctx->msg_count < ctx->layout->msg_count); + m->layout = ctx->layout->msgs[ctx->msg_count++]; + UPB_ASSERT(n_field == m->layout->field_count); + } else { + /* Allocate now (to allow cross-linking), populate later. */ + m->layout = _upb_DefBuilder_Alloc( + ctx, sizeof(*m->layout) + sizeof(_upb_FastTable_Entry)); + } + UPB_DEF_SET_OPTIONS(m->opts, DescriptorProto, MessageOptions, msg_proto); m->oneof_count = n_oneof; @@ -10918,7 +10836,7 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, const UPB_DESC(FieldDescriptorProto)* const* exts = UPB_DESC(DescriptorProto_extension)(msg_proto, &n_ext); m->nested_ext_count = n_ext; - m->nested_exts = _upb_Extensions_New(ctx, n_ext, exts, m->full_name, m); + m->nested_exts = _upb_FieldDefs_New(ctx, n_ext, exts, m->full_name, m, NULL); const UPB_DESC(DescriptorProto)* const* msgs = UPB_DESC(DescriptorProto_nested_type)(msg_proto, &n_msg); @@ -11325,6 +11243,7 @@ static void create_service(upb_DefBuilder* ctx, s->file = _upb_DefBuilder_File(ctx); name = UPB_DESC(ServiceDescriptorProto_name)(svc_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); const char* package = _upb_FileDef_RawPackage(s->file); s->full_name = _upb_DefBuilder_MakeFullName(ctx, package, name); _upb_DefBuilder_Add(ctx, s->full_name, @@ -14092,15 +14011,6 @@ static void encode_msgset_item(upb_encstate* e, encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_StartGroup); } -static void encode_ext(upb_encstate* e, const upb_Message_Extension* ext, - bool is_message_set) { - if (UPB_UNLIKELY(is_message_set)) { - encode_msgset_item(e, ext); - } else { - encode_field(e, &ext->data, &ext->ext->sub, &ext->ext->field); - } -} - static void encode_message(upb_encstate* e, const upb_Message* msg, const upb_MiniTable* m, size_t* size) { size_t pre_len = e->limit - e->ptr; @@ -14130,17 +14040,12 @@ static void encode_message(upb_encstate* e, const upb_Message* msg, size_t ext_count; const upb_Message_Extension* ext = _upb_Message_Getexts(msg, &ext_count); if (ext_count) { - if (e->options & kUpb_EncodeOption_Deterministic) { - _upb_sortedmap sorted; - _upb_mapsorter_pushexts(&e->sorter, ext, ext_count, &sorted); - while (_upb_sortedmap_nextext(&e->sorter, &sorted, &ext)) { - encode_ext(e, ext, m->ext == kUpb_ExtMode_IsMessageSet); - } - _upb_mapsorter_popmap(&e->sorter, &sorted); - } else { - const upb_Message_Extension* end = ext + ext_count; - for (; ext != end; ext++) { - encode_ext(e, ext, m->ext == kUpb_ExtMode_IsMessageSet); + const upb_Message_Extension* end = ext + ext_count; + for (; ext != end; ext++) { + if (UPB_UNLIKELY(m->ext == kUpb_ExtMode_IsMessageSet)) { + encode_msgset_item(e, ext); + } else { + encode_field(e, &ext->data, &ext->ext->sub, &ext->ext->field); } } } diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 3de3e6d34c00..6b554cc3427e 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -1422,6 +1422,179 @@ UPB_INLINE int upb_Log2CeilingSize(int x) { return 1 << upb_Log2Ceiling(x); } #include +#ifndef UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ +#define UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ + + +// Must be last. + +struct upb_Decoder; +typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, + upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t data); +typedef struct { + uint64_t field_data; + _upb_FieldParser* field_parser; +} _upb_FastTable_Entry; + +typedef enum { + kUpb_ExtMode_NonExtendable = 0, // Non-extendable message. + kUpb_ExtMode_Extendable = 1, // Normal extendable message. + kUpb_ExtMode_IsMessageSet = 2, // MessageSet message. + kUpb_ExtMode_IsMessageSet_ITEM = + 3, // MessageSet item (temporary only, see decode.c) + + // During table building we steal a bit to indicate that the message is a map + // entry. *Only* used during table building! + kUpb_ExtMode_IsMapEntry = 4, +} upb_ExtMode; + +// upb_MiniTable represents the memory layout of a given upb_MessageDef. +// The members are public so generated code can initialize them, +// but users MUST NOT directly read or write any of its members. +struct upb_MiniTable { + const upb_MiniTableSub* subs; + const upb_MiniTableField* fields; + + // Must be aligned to sizeof(void*). Doesn't include internal members like + // unknown fields, extension dict, pointer to msglayout, etc. + uint16_t size; + + uint16_t field_count; + uint8_t ext; // upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 + uint8_t dense_below; + uint8_t table_mask; + uint8_t required_count; // Required fields have the lowest hasbits. + + // To statically initialize the tables of variable length, we need a flexible + // array member, and we need to compile in gnu99 mode (constant initialization + // of flexible array members is a GNU extension, not in C99 unfortunately. + _upb_FastTable_Entry fasttable[]; +}; + +// Map entries aren't actually stored for map fields, they are only used during +// parsing. For parsing, it helps a lot if all map entry messages have the same +// layout. The layout code in mini_table/decode.c will ensure that all map +// entries have this layout. +// +// Note that users can and do create map entries directly, which will also use +// this layout. +// +// NOTE: sync with mini_table/decode.c. +typedef struct { + // We only need 2 hasbits max, but due to alignment we'll use 8 bytes here, + // and the uint64_t helps make this clear. + uint64_t hasbits; + union { + upb_StringView str; // For str/bytes. + upb_value val; // For all other types. + } k; + union { + upb_StringView str; // For str/bytes. + upb_value val; // For all other types. + } v; +} upb_MapEntryData; + +typedef struct { + void* internal_data; + upb_MapEntryData data; +} upb_MapEntry; + +#ifdef __cplusplus +extern "C" { +#endif + +// Computes a bitmask in which the |l->required_count| lowest bits are set, +// except that we skip the lowest bit (because upb never uses hasbit 0). +// +// Sample output: +// requiredmask(1) => 0b10 (0x2) +// requiredmask(5) => 0b111110 (0x3e) +UPB_INLINE uint64_t upb_MiniTable_requiredmask(const upb_MiniTable* l) { + int n = l->required_count; + assert(0 < n && n <= 63); + return ((1ULL << n) - 1) << 1; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ */ + +// Must be last. + +#ifdef __cplusplus +extern "C" { +#endif + +// _upb_mapsorter sorts maps and provides ordered iteration over the entries. +// Since maps can be recursive (map values can be messages which contain other +// maps), _upb_mapsorter can contain a stack of maps. + +typedef struct { + upb_tabent const** entries; + int size; + int cap; +} _upb_mapsorter; + +typedef struct { + int start; + int pos; + int end; +} _upb_sortedmap; + +UPB_INLINE void _upb_mapsorter_init(_upb_mapsorter* s) { + s->entries = NULL; + s->size = 0; + s->cap = 0; +} + +UPB_INLINE void _upb_mapsorter_destroy(_upb_mapsorter* s) { + if (s->entries) free(s->entries); +} + +UPB_INLINE bool _upb_sortedmap_next(_upb_mapsorter* s, const upb_Map* map, + _upb_sortedmap* sorted, upb_MapEntry* ent) { + if (sorted->pos == sorted->end) return false; + const upb_tabent* tabent = s->entries[sorted->pos++]; + upb_StringView key = upb_tabstrview(tabent->key); + _upb_map_fromkey(key, &ent->data.k, map->key_size); + upb_value val = {tabent->val.val}; + _upb_map_fromvalue(val, &ent->data.v, map->val_size); + return true; +} + +UPB_INLINE void _upb_mapsorter_popmap(_upb_mapsorter* s, + _upb_sortedmap* sorted) { + s->size = sorted->start; +} + +bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, + const upb_Map* map, _upb_sortedmap* sorted); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_COLLECTIONS_MAP_SORTER_INTERNAL_H_ */ + +/* +** Our memory representation for parsing tables and messages themselves. +** Functions in this file are used by generated code and possibly reflection. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MESSAGE_INTERNAL_H_ +#define UPB_MESSAGE_INTERNAL_H_ + +#include +#include + + #ifndef UPB_MESSAGE_EXTENSION_INTERNAL_H_ #define UPB_MESSAGE_EXTENSION_INTERNAL_H_ @@ -1694,191 +1867,6 @@ const upb_Message_Extension* _upb_Message_Getext( #endif /* UPB_MESSAGE_EXTENSION_INTERNAL_H_ */ -#ifndef UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ -#define UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ - - -// Must be last. - -struct upb_Decoder; -typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, - upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t data); -typedef struct { - uint64_t field_data; - _upb_FieldParser* field_parser; -} _upb_FastTable_Entry; - -typedef enum { - kUpb_ExtMode_NonExtendable = 0, // Non-extendable message. - kUpb_ExtMode_Extendable = 1, // Normal extendable message. - kUpb_ExtMode_IsMessageSet = 2, // MessageSet message. - kUpb_ExtMode_IsMessageSet_ITEM = - 3, // MessageSet item (temporary only, see decode.c) - - // During table building we steal a bit to indicate that the message is a map - // entry. *Only* used during table building! - kUpb_ExtMode_IsMapEntry = 4, -} upb_ExtMode; - -// upb_MiniTable represents the memory layout of a given upb_MessageDef. -// The members are public so generated code can initialize them, -// but users MUST NOT directly read or write any of its members. -struct upb_MiniTable { - const upb_MiniTableSub* subs; - const upb_MiniTableField* fields; - - // Must be aligned to sizeof(void*). Doesn't include internal members like - // unknown fields, extension dict, pointer to msglayout, etc. - uint16_t size; - - uint16_t field_count; - uint8_t ext; // upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 - uint8_t dense_below; - uint8_t table_mask; - uint8_t required_count; // Required fields have the lowest hasbits. - - // To statically initialize the tables of variable length, we need a flexible - // array member, and we need to compile in gnu99 mode (constant initialization - // of flexible array members is a GNU extension, not in C99 unfortunately. - _upb_FastTable_Entry fasttable[]; -}; - -// Map entries aren't actually stored for map fields, they are only used during -// parsing. For parsing, it helps a lot if all map entry messages have the same -// layout. The layout code in mini_table/decode.c will ensure that all map -// entries have this layout. -// -// Note that users can and do create map entries directly, which will also use -// this layout. -// -// NOTE: sync with mini_table/decode.c. -typedef struct { - // We only need 2 hasbits max, but due to alignment we'll use 8 bytes here, - // and the uint64_t helps make this clear. - uint64_t hasbits; - union { - upb_StringView str; // For str/bytes. - upb_value val; // For all other types. - } k; - union { - upb_StringView str; // For str/bytes. - upb_value val; // For all other types. - } v; -} upb_MapEntryData; - -typedef struct { - void* internal_data; - upb_MapEntryData data; -} upb_MapEntry; - -#ifdef __cplusplus -extern "C" { -#endif - -// Computes a bitmask in which the |l->required_count| lowest bits are set, -// except that we skip the lowest bit (because upb never uses hasbit 0). -// -// Sample output: -// requiredmask(1) => 0b10 (0x2) -// requiredmask(5) => 0b111110 (0x3e) -UPB_INLINE uint64_t upb_MiniTable_requiredmask(const upb_MiniTable* l) { - int n = l->required_count; - assert(0 < n && n <= 63); - return ((1ULL << n) - 1) << 1; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ */ - -// Must be last. - -#ifdef __cplusplus -extern "C" { -#endif - -// _upb_mapsorter sorts maps and provides ordered iteration over the entries. -// Since maps can be recursive (map values can be messages which contain other -// maps), _upb_mapsorter can contain a stack of maps. - -typedef struct { - void const** entries; - int size; - int cap; -} _upb_mapsorter; - -typedef struct { - int start; - int pos; - int end; -} _upb_sortedmap; - -UPB_INLINE void _upb_mapsorter_init(_upb_mapsorter* s) { - s->entries = NULL; - s->size = 0; - s->cap = 0; -} - -UPB_INLINE void _upb_mapsorter_destroy(_upb_mapsorter* s) { - if (s->entries) free(s->entries); -} - -UPB_INLINE bool _upb_sortedmap_next(_upb_mapsorter* s, const upb_Map* map, - _upb_sortedmap* sorted, upb_MapEntry* ent) { - if (sorted->pos == sorted->end) return false; - const upb_tabent* tabent = (const upb_tabent*)s->entries[sorted->pos++]; - upb_StringView key = upb_tabstrview(tabent->key); - _upb_map_fromkey(key, &ent->data.k, map->key_size); - upb_value val = {tabent->val.val}; - _upb_map_fromvalue(val, &ent->data.v, map->val_size); - return true; -} - -UPB_INLINE bool _upb_sortedmap_nextext(_upb_mapsorter* s, - _upb_sortedmap* sorted, - const upb_Message_Extension** ext) { - if (sorted->pos == sorted->end) return false; - *ext = (const upb_Message_Extension*)s->entries[sorted->pos++]; - return true; -} - -UPB_INLINE void _upb_mapsorter_popmap(_upb_mapsorter* s, - _upb_sortedmap* sorted) { - s->size = sorted->start; -} - -bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, - const upb_Map* map, _upb_sortedmap* sorted); - -bool _upb_mapsorter_pushexts(_upb_mapsorter* s, - const upb_Message_Extension* exts, size_t count, - _upb_sortedmap* sorted); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_COLLECTIONS_MAP_SORTER_INTERNAL_H_ */ - -/* -** Our memory representation for parsing tables and messages themselves. -** Functions in this file are used by generated code and possibly reflection. -** -** The definitions in this file are internal to upb. -**/ - -#ifndef UPB_MESSAGE_INTERNAL_H_ -#define UPB_MESSAGE_INTERNAL_H_ - -#include -#include - - #ifndef UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ #define UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ @@ -8338,7 +8326,6 @@ bool upb_FileDef_HasOptions(const upb_FileDef* f); const char* upb_FileDef_Name(const upb_FileDef* f); const UPB_DESC(FileOptions) * upb_FileDef_Options(const upb_FileDef* f); const char* upb_FileDef_Package(const upb_FileDef* f); -const char* upb_FileDef_Edition(const upb_FileDef* f); const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); const upb_FileDef* upb_FileDef_PublicDependency(const upb_FileDef* f, int i); @@ -8724,7 +8711,7 @@ size_t _upb_DefPool_BytesLoaded(const upb_DefPool* s); upb_ExtensionRegistry* _upb_DefPool_ExtReg(const upb_DefPool* s); bool _upb_DefPool_InsertExt(upb_DefPool* s, const upb_MiniTableExtension* ext, - const upb_FieldDef* f); + upb_FieldDef* f); bool _upb_DefPool_InsertSym(upb_DefPool* s, upb_StringView sym, upb_value v, upb_Status* status); bool _upb_DefPool_LookupSym(const upb_DefPool* s, const char* sym, size_t size, @@ -10210,6 +10197,23 @@ UPB_INLINE upb_FileDef* _upb_DefBuilder_File(const upb_DefBuilder* ctx) { void _upb_DefBuilder_CheckIdentSlow(upb_DefBuilder* ctx, upb_StringView name, bool full); +// Verify a relative identifier string. The loop is branchless for speed. +UPB_INLINE void _upb_DefBuilder_CheckIdentNotFull(upb_DefBuilder* ctx, + upb_StringView name) { + bool good = name.size > 0; + + for (size_t i = 0; i < name.size; i++) { + const char c = name.data[i]; + const char d = c | 0x20; // force lowercase + const bool is_alpha = (('a' <= d) & (d <= 'z')) | (c == '_'); + const bool is_numer = ('0' <= c) & (c <= '9') & (i != 0); + + good &= is_alpha | is_numer; + } + + if (!good) _upb_DefBuilder_CheckIdentSlow(ctx, name, false); +} + // Verify a full identifier string. This is slightly more complicated than // verifying a relative identifier string because we must track '.' chars. UPB_INLINE void _upb_DefBuilder_CheckIdentFull(upb_DefBuilder* ctx, @@ -10313,14 +10317,6 @@ int _upb_FieldDef_LayoutIndex(const upb_FieldDef* f); uint64_t _upb_FieldDef_Modifiers(const upb_FieldDef* f); void _upb_FieldDef_Resolve(upb_DefBuilder* ctx, const char* prefix, upb_FieldDef* f); -void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, - const upb_FieldDef* f); - -// Allocate and initialize an array of |n| extensions (field defs). -upb_FieldDef* _upb_Extensions_New( - upb_DefBuilder* ctx, int n, - const UPB_DESC(FieldDescriptorProto) * const* protos, const char* prefix, - upb_MessageDef* m); // Allocate and initialize an array of |n| field defs. upb_FieldDef* _upb_FieldDefs_New( @@ -10385,7 +10381,6 @@ bool _upb_MessageDef_Insert(upb_MessageDef* m, const char* name, size_t size, void _upb_MessageDef_InsertField(upb_DefBuilder* ctx, upb_MessageDef* m, const upb_FieldDef* f); bool _upb_MessageDef_IsValidExtensionNumber(const upb_MessageDef* m, int n); -void _upb_MessageDef_CreateMiniTable(upb_DefBuilder* ctx, upb_MessageDef* m); void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m); void _upb_MessageDef_Resolve(upb_DefBuilder* ctx, upb_MessageDef* m); diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9f6436e04617..fee1e9c1b15e 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -149,7 +149,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "1fb480bc76bc0e331564d672e60b97a388aa3f76", - sha256 = "a5396af1eeb1144c17cec454332997374d32854007f74390b154ec7255b49ded", + commit = "662497f1d3dcced2bba1620cea9aae8b484bd3cd", + sha256 = "57c87ca4145d2cbc162a6c613b114b9325b577f4f6525bd78747a34b3d03627c", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index bda70b21c33e..6fbf78bc0282 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -641,12 +641,14 @@ static int (*const compar[kUpb_FieldType_SizeOf])(const void*, const void*) = { [kUpb_FieldType_Bytes] = _upb_mapsorter_cmpstr, }; -static bool _upb_mapsorter_resize(_upb_mapsorter* s, _upb_sortedmap* sorted, - int size) { +bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, + const upb_Map* map, _upb_sortedmap* sorted) { + int map_size = _upb_Map_Size(map); sorted->start = s->size; sorted->pos = sorted->start; - sorted->end = sorted->start + size; + sorted->end = sorted->start + map_size; + // Grow s->entries if necessary. if (sorted->end > s->cap) { s->cap = upb_Log2CeilingSize(sorted->end); s->entries = realloc(s->entries, s->cap * sizeof(*s->entries)); @@ -654,17 +656,9 @@ static bool _upb_mapsorter_resize(_upb_mapsorter* s, _upb_sortedmap* sorted, } s->size = sorted->end; - return true; -} - -bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, - const upb_Map* map, _upb_sortedmap* sorted) { - int map_size = _upb_Map_Size(map); - - if (!_upb_mapsorter_resize(s, sorted, map_size)) return false; // Copy non-empty entries from the table to s->entries. - const void** dst = &s->entries[sorted->start]; + upb_tabent const** dst = &s->entries[sorted->start]; const upb_tabent* src = map->table.t.entries; const upb_tabent* end = src + upb_table_size(&map->table.t); for (; src < end; src++) { @@ -680,29 +674,6 @@ bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, compar[key_type]); return true; } - -static int _upb_mapsorter_cmpext(const void* _a, const void* _b) { - const upb_Message_Extension* const* a = _a; - const upb_Message_Extension* const* b = _b; - uint32_t a_num = (*a)->ext->field.number; - uint32_t b_num = (*b)->ext->field.number; - assert(a_num != b_num); - return a_num < b_num ? -1 : 1; -} - -bool _upb_mapsorter_pushexts(_upb_mapsorter* s, - const upb_Message_Extension* exts, size_t count, - _upb_sortedmap* sorted) { - if (!_upb_mapsorter_resize(s, sorted, count)) return false; - - for (size_t i = 0; i < count; i++) { - s->entries[sorted->start + i] = &exts[i]; - } - - qsort(&s->entries[sorted->start], count, sizeof(*s->entries), - _upb_mapsorter_cmpext); - return true; -} /* This file was generated by upbc (the upb compiler) from the input * file: * @@ -1891,7 +1862,7 @@ char* upb_strdup2(const char* s, size_t len, upb_Arena* a) { n = len + 1; p = upb_Arena_Malloc(a, n); if (p) { - if (len != 0) memcpy(p, s, len); + memcpy(p, s, len); p[len] = 0; } return p; @@ -7227,27 +7198,9 @@ void _upb_DefBuilder_OomErr(upb_DefBuilder* ctx) { _upb_DefBuilder_FailJmp(ctx); } -// Verify a relative identifier string. The loop is branchless for speed. -static void _upb_DefBuilder_CheckIdentNotFull(upb_DefBuilder* ctx, - upb_StringView name) { - bool good = name.size > 0; - - for (size_t i = 0; i < name.size; i++) { - const char c = name.data[i]; - const char d = c | 0x20; // force lowercase - const bool is_alpha = (('a' <= d) & (d <= 'z')) | (c == '_'); - const bool is_numer = ('0' <= c) & (c <= '9') & (i != 0); - - good &= is_alpha | is_numer; - } - - if (!good) _upb_DefBuilder_CheckIdentSlow(ctx, name, false); -} - const char* _upb_DefBuilder_MakeFullName(upb_DefBuilder* ctx, const char* prefix, upb_StringView name) { - _upb_DefBuilder_CheckIdentNotFull(ctx, name); if (prefix) { // ret = prefix + '.' + name; size_t n = strlen(prefix); @@ -7363,7 +7316,7 @@ static bool TryGetChar(const char** src, const char* end, char* ch) { return true; } -static int TryGetHexDigit(const char** src, const char* end) { +static char TryGetHexDigit(const char** src, const char* end) { char ch; if (!TryGetChar(src, end, &ch)) return -1; if ('0' <= ch && ch <= '9') { @@ -7380,10 +7333,10 @@ static int TryGetHexDigit(const char** src, const char* end) { static char upb_DefBuilder_ParseHexEscape(upb_DefBuilder* ctx, const upb_FieldDef* f, const char** src, const char* end) { - int hex_digit = TryGetHexDigit(src, end); + char hex_digit = TryGetHexDigit(src, end); if (hex_digit < 0) { _upb_DefBuilder_Errf( - ctx, "\\x must be followed by at least one hex digit (field='%s')", + ctx, "\\x cannot be followed by non-hex digit in field '%s' default", upb_FieldDef_FullName(f)); return 0; } @@ -7559,7 +7512,7 @@ upb_DefPool* upb_DefPool_New(void) { } bool _upb_DefPool_InsertExt(upb_DefPool* s, const upb_MiniTableExtension* ext, - const upb_FieldDef* f) { + upb_FieldDef* f) { return upb_inttable_insert(&s->exts, (uintptr_t)ext, upb_value_constptr(f), s->arena); } @@ -7791,6 +7744,12 @@ static const upb_FileDef* _upb_DefPool_AddFile( const upb_MiniTableFile* layout, upb_Status* status) { const upb_StringView name = UPB_DESC(FileDescriptorProto_name)(file_proto); + if (name.size == 0) { + upb_Status_SetErrorFormat(status, + "missing name in google_protobuf_FileDescriptorProto"); + return NULL; + } + // Determine whether we already know about this file. { upb_value v; @@ -8182,6 +8141,7 @@ static void create_enumdef(upb_DefBuilder* ctx, const char* prefix, e->file = _upb_DefBuilder_File(ctx); name = UPB_DESC(EnumDescriptorProto_name)(enum_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); e->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); _upb_DefBuilder_Add(ctx, e->full_name, @@ -8991,14 +8951,7 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, } const upb_StringView name = UPB_DESC(FieldDescriptorProto_name)(field_proto); - - f->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); - f->label_ = (int)UPB_DESC(FieldDescriptorProto_label)(field_proto); - f->number_ = UPB_DESC(FieldDescriptorProto_number)(field_proto); - f->is_proto3_optional = - UPB_DESC(FieldDescriptorProto_proto3_optional)(field_proto); - f->msgdef = m; - f->scope.oneof = NULL; + _upb_DefBuilder_CheckIdentNotFull(ctx, name); f->has_json_name = UPB_DESC(FieldDescriptorProto_has_json_name)(field_proto); if (f->has_json_name) { @@ -9010,6 +8963,14 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, } if (!f->json_name) _upb_DefBuilder_OomErr(ctx); + f->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); + f->label_ = (int)UPB_DESC(FieldDescriptorProto_label)(field_proto); + f->number_ = UPB_DESC(FieldDescriptorProto_number)(field_proto); + f->is_proto3_optional = + UPB_DESC(FieldDescriptorProto_proto3_optional)(field_proto); + f->msgdef = m; + f->scope.oneof = NULL; + const bool has_type = UPB_DESC(FieldDescriptorProto_has_type)(field_proto); const bool has_type_name = UPB_DESC(FieldDescriptorProto_has_type_name)(field_proto); @@ -9139,24 +9100,19 @@ static void _upb_FieldDef_CreateNotExt(upb_DefBuilder* ctx, const char* prefix, } _upb_MessageDef_InsertField(ctx, m, f); -} -upb_FieldDef* _upb_Extensions_New( - upb_DefBuilder* ctx, int n, - const UPB_DESC(FieldDescriptorProto) * const* protos, const char* prefix, - upb_MessageDef* m) { - _upb_DefType_CheckPadding(sizeof(upb_FieldDef)); - upb_FieldDef* defs = - (upb_FieldDef*)_upb_DefBuilder_Alloc(ctx, sizeof(upb_FieldDef) * n); + if (!ctx->layout) return; - for (int i = 0; i < n; i++) { - upb_FieldDef* f = &defs[i]; - - _upb_FieldDef_CreateExt(ctx, prefix, protos[i], m, f); - f->index_ = i; + const upb_MiniTable* mt = upb_MessageDef_MiniTable(m); + const upb_MiniTableField* fields = mt->fields; + for (int i = 0; i < mt->field_count; i++) { + if (fields[i].number == f->number_) { + f->layout_index = i; + return; + } } - return defs; + UPB_ASSERT(false); // It should be impossible to reach this point. } upb_FieldDef* _upb_FieldDefs_New( @@ -9167,23 +9123,28 @@ upb_FieldDef* _upb_FieldDefs_New( upb_FieldDef* defs = (upb_FieldDef*)_upb_DefBuilder_Alloc(ctx, sizeof(upb_FieldDef) * n); - uint32_t previous = 0; - for (int i = 0; i < n; i++) { - upb_FieldDef* f = &defs[i]; - - _upb_FieldDef_CreateNotExt(ctx, prefix, protos[i], m, f); - f->index_ = i; - if (!ctx->layout) { - // Speculate that the def fields are sorted. We will always sort the - // MiniTable fields, so if defs are sorted then indices will match. - // - // If this is incorrect, we will overwrite later. - f->layout_index = i; + // If we are creating extensions then is_sorted will be NULL. + // If we are not creating extensions then is_sorted will be non-NULL. + if (is_sorted) { + uint32_t previous = 0; + for (int i = 0; i < n; i++) { + upb_FieldDef* f = &defs[i]; + + _upb_FieldDef_CreateNotExt(ctx, prefix, protos[i], m, f); + f->index_ = i; + if (!ctx->layout) f->layout_index = i; + + const uint32_t current = f->number_; + if (previous > current) *is_sorted = false; + previous = current; } + } else { + for (int i = 0; i < n; i++) { + upb_FieldDef* f = &defs[i]; - const uint32_t current = f->number_; - if (previous > current) *is_sorted = false; - previous = current; + _upb_FieldDef_CreateExt(ctx, prefix, protos[i], m, f); + f->index_ = i; + } } return defs; @@ -9239,9 +9200,6 @@ static int _upb_FieldDef_Compare(const void* p1, const void* p2) { return (v1 < v2) ? -1 : (v1 > v2); } -// _upb_FieldDefs_Sorted() is mostly a pure function of its inputs, but has one -// critical side effect that we depend on: it sets layout_index appropriately -// for non-sorted lists of fields. const upb_FieldDef** _upb_FieldDefs_Sorted(const upb_FieldDef* f, int n, upb_Arena* a) { // TODO(salo): Replace this arena alloc with a persistent scratch buffer. @@ -9299,10 +9257,7 @@ static void resolve_extension(upb_DefBuilder* ctx, const char* prefix, "field number %u in extension %s has no extension range in message %s", (unsigned)f->number_, f->full_name, upb_MessageDef_FullName(m)); } -} -void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, - const upb_FieldDef* f) { const upb_MiniTableExtension* ext = _upb_FieldDef_ExtensionMiniTable(f); if (ctx->layout) { @@ -9321,8 +9276,8 @@ void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, sub.subenum = _upb_EnumDef_MiniTable(f->sub.enumdef); } bool ok2 = upb_MiniTableExtension_Build(desc.data, desc.size, mut_ext, - upb_MessageDef_MiniTable(f->msgdef), - sub, ctx->status); + upb_MessageDef_MiniTable(m), sub, + ctx->status); if (!ok2) _upb_DefBuilder_Errf(ctx, "Could not build extension mini table"); } @@ -9379,7 +9334,6 @@ struct upb_FileDef { const UPB_DESC(FileOptions) * opts; const char* name; const char* package; - const char* edition; const upb_FileDef** deps; const int32_t* public_deps; @@ -9416,10 +9370,6 @@ const char* upb_FileDef_Package(const upb_FileDef* f) { return f->package ? f->package : ""; } -const char* upb_FileDef_Edition(const upb_FileDef* f) { - return f->edition ? f->edition : ""; -} - const char* _upb_FileDef_RawPackage(const upb_FileDef* f) { return f->package; } upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f) { return f->syntax; } @@ -9567,14 +9517,13 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, } } - upb_StringView name = UPB_DESC(FileDescriptorProto_name)(file_proto); - file->name = strviewdup(ctx, name); - if (strlen(file->name) != name.size) { - _upb_DefBuilder_Errf(ctx, "File name contained embedded NULL"); + if (!UPB_DESC(FileDescriptorProto_has_name)(file_proto)) { + _upb_DefBuilder_Errf(ctx, "File has no name"); } - upb_StringView package = UPB_DESC(FileDescriptorProto_package)(file_proto); + file->name = strviewdup(ctx, UPB_DESC(FileDescriptorProto_name)(file_proto)); + upb_StringView package = UPB_DESC(FileDescriptorProto_package)(file_proto); if (package.size) { _upb_DefBuilder_CheckIdentFull(ctx, package); file->package = strviewdup(ctx, package); @@ -9582,18 +9531,6 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, file->package = NULL; } - upb_StringView edition = UPB_DESC(FileDescriptorProto_edition)(file_proto); - - if (edition.size == 0) { - file->edition = NULL; - } else { - // TODO(b/267770604): How should we validate this? - file->edition = strviewdup(ctx, edition); - if (strlen(file->edition) != edition.size) { - _upb_DefBuilder_Errf(ctx, "Edition name contained embedded NULL"); - } - } - if (UPB_DESC(FileDescriptorProto_has_syntax)(file_proto)) { upb_StringView syntax = UPB_DESC(FileDescriptorProto_syntax)(file_proto); @@ -9662,7 +9599,8 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, // Create extensions. exts = UPB_DESC(FileDescriptorProto_extension)(file_proto, &n); file->top_lvl_ext_count = n; - file->top_lvl_exts = _upb_Extensions_New(ctx, n, exts, file->package, NULL); + file->top_lvl_exts = + _upb_FieldDefs_New(ctx, n, exts, file->package, NULL, NULL); // Create messages. msgs = UPB_DESC(FileDescriptorProto_message_type)(file_proto, &n); @@ -9686,19 +9624,11 @@ void _upb_FileDef_Create(upb_DefBuilder* ctx, _upb_FieldDef_Resolve(ctx, file->package, f); } - for (int i = 0; i < file->top_lvl_msg_count; i++) { - upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); - _upb_MessageDef_CreateMiniTable(ctx, (upb_MessageDef*)m); - } - - for (int i = 0; i < file->top_lvl_ext_count; i++) { - upb_FieldDef* f = (upb_FieldDef*)upb_FileDef_TopLevelExtension(file, i); - _upb_FieldDef_BuildMiniTableExtension(ctx, f); - } - - for (int i = 0; i < file->top_lvl_msg_count; i++) { - upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); - _upb_MessageDef_LinkMiniTable(ctx, m); + if (!ctx->layout) { + for (int i = 0; i < file->top_lvl_msg_count; i++) { + upb_MessageDef* m = (upb_MessageDef*)upb_FileDef_TopLevelMessage(file, i); + _upb_MessageDef_LinkMiniTable(ctx, m); + } } if (file->ext_count) { @@ -10211,8 +10141,6 @@ bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m) { static upb_MiniTable* _upb_MessageDef_MakeMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m) { upb_StringView desc; - // Note: this will assign layout_index for fields, so upb_FieldDef_MiniTable() - // is safe to call only after this call. bool ok = upb_MessageDef_MiniDescriptorEncode(m, ctx->tmp_arena, &desc); if (!ok) _upb_DefBuilder_OomErr(ctx); @@ -10232,6 +10160,23 @@ void _upb_MessageDef_Resolve(upb_DefBuilder* ctx, upb_MessageDef* m) { _upb_FieldDef_Resolve(ctx, m->full_name, f); } + if (!ctx->layout) { + m->layout = _upb_MessageDef_MakeMiniTable(ctx, m); + if (!m->layout) _upb_DefBuilder_OomErr(ctx); + } + +#ifndef NDEBUG + for (int i = 0; i < m->field_count; i++) { + const upb_FieldDef* f = upb_MessageDef_Field(m, i); + const int layout_index = _upb_FieldDef_LayoutIndex(f); + UPB_ASSERT(layout_index < m->layout->field_count); + const upb_MiniTableField* mt_f = &m->layout->fields[layout_index]; + UPB_ASSERT(upb_FieldDef_Type(f) == upb_MiniTableField_Type(mt_f)); + UPB_ASSERT(upb_FieldDef_HasPresence(f) == + upb_MiniTableField_HasPresence(mt_f)); + } +#endif + m->in_message_set = false; for (int i = 0; i < upb_MessageDef_NestedExtensionCount(m); i++) { upb_FieldDef* ext = (upb_FieldDef*)upb_MessageDef_NestedExtension(m, i); @@ -10294,39 +10239,8 @@ void _upb_MessageDef_InsertField(upb_DefBuilder* ctx, upb_MessageDef* m, if (!ok) _upb_DefBuilder_OomErr(ctx); } -void _upb_MessageDef_CreateMiniTable(upb_DefBuilder* ctx, upb_MessageDef* m) { - if (ctx->layout == NULL) { - m->layout = _upb_MessageDef_MakeMiniTable(ctx, m); - } else { - UPB_ASSERT(ctx->msg_count < ctx->layout->msg_count); - m->layout = ctx->layout->msgs[ctx->msg_count++]; - UPB_ASSERT(m->field_count == m->layout->field_count); - - // We don't need the result of this call, but it will assign layout_index - // for all the fields in O(n lg n) time. - _upb_FieldDefs_Sorted(m->fields, m->field_count, ctx->tmp_arena); - } - - for (int i = 0; i < m->nested_msg_count; i++) { - upb_MessageDef* nested = - (upb_MessageDef*)upb_MessageDef_NestedMessage(m, i); - _upb_MessageDef_CreateMiniTable(ctx, nested); - } -} - void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m) { - for (int i = 0; i < upb_MessageDef_NestedExtensionCount(m); i++) { - const upb_FieldDef* ext = upb_MessageDef_NestedExtension(m, i); - _upb_FieldDef_BuildMiniTableExtension(ctx, ext); - } - - for (int i = 0; i < m->nested_msg_count; i++) { - _upb_MessageDef_LinkMiniTable(ctx, upb_MessageDef_NestedMessage(m, i)); - } - - if (ctx->layout) return; - for (int i = 0; i < m->field_count; i++) { const upb_FieldDef* f = upb_MessageDef_Field(m, i); const upb_MessageDef* sub_m = upb_FieldDef_MessageSubDef(f); @@ -10354,17 +10268,9 @@ void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, } } -#ifndef NDEBUG - for (int i = 0; i < m->field_count; i++) { - const upb_FieldDef* f = upb_MessageDef_Field(m, i); - const int layout_index = _upb_FieldDef_LayoutIndex(f); - UPB_ASSERT(layout_index < m->layout->field_count); - const upb_MiniTableField* mt_f = &m->layout->fields[layout_index]; - UPB_ASSERT(upb_FieldDef_Type(f) == upb_MiniTableField_Type(mt_f)); - UPB_ASSERT(upb_FieldDef_HasPresence(f) == - upb_MiniTableField_HasPresence(mt_f)); + for (int i = 0; i < m->nested_msg_count; i++) { + _upb_MessageDef_LinkMiniTable(ctx, upb_MessageDef_NestedMessage(m, i)); } -#endif } static uint64_t _upb_MessageDef_Modifiers(const upb_MessageDef* m) { @@ -10497,6 +10403,7 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, m->is_sorted = true; name = UPB_DESC(DescriptorProto_name)(msg_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); m->full_name = _upb_DefBuilder_MakeFullName(ctx, prefix, name); _upb_DefBuilder_Add(ctx, m->full_name, _upb_DefType_Pack(m, UPB_DEFTYPE_MSG)); @@ -10515,6 +10422,17 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, ok = upb_strtable_init(&m->ntof, n_oneof + n_field, ctx->arena); if (!ok) _upb_DefBuilder_OomErr(ctx); + if (ctx->layout) { + /* create_fielddef() below depends on this being set. */ + UPB_ASSERT(ctx->msg_count < ctx->layout->msg_count); + m->layout = ctx->layout->msgs[ctx->msg_count++]; + UPB_ASSERT(n_field == m->layout->field_count); + } else { + /* Allocate now (to allow cross-linking), populate later. */ + m->layout = _upb_DefBuilder_Alloc( + ctx, sizeof(*m->layout) + sizeof(_upb_FastTable_Entry)); + } + UPB_DEF_SET_OPTIONS(m->opts, DescriptorProto, MessageOptions, msg_proto); m->oneof_count = n_oneof; @@ -10555,7 +10473,7 @@ static void create_msgdef(upb_DefBuilder* ctx, const char* prefix, const UPB_DESC(FieldDescriptorProto)* const* exts = UPB_DESC(DescriptorProto_extension)(msg_proto, &n_ext); m->nested_ext_count = n_ext; - m->nested_exts = _upb_Extensions_New(ctx, n_ext, exts, m->full_name, m); + m->nested_exts = _upb_FieldDefs_New(ctx, n_ext, exts, m->full_name, m, NULL); const UPB_DESC(DescriptorProto)* const* msgs = UPB_DESC(DescriptorProto_nested_type)(msg_proto, &n_msg); @@ -10962,6 +10880,7 @@ static void create_service(upb_DefBuilder* ctx, s->file = _upb_DefBuilder_File(ctx); name = UPB_DESC(ServiceDescriptorProto_name)(svc_proto); + _upb_DefBuilder_CheckIdentNotFull(ctx, name); const char* package = _upb_FileDef_RawPackage(s->file); s->full_name = _upb_DefBuilder_MakeFullName(ctx, package, name); _upb_DefBuilder_Add(ctx, s->full_name, @@ -13729,15 +13648,6 @@ static void encode_msgset_item(upb_encstate* e, encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_StartGroup); } -static void encode_ext(upb_encstate* e, const upb_Message_Extension* ext, - bool is_message_set) { - if (UPB_UNLIKELY(is_message_set)) { - encode_msgset_item(e, ext); - } else { - encode_field(e, &ext->data, &ext->ext->sub, &ext->ext->field); - } -} - static void encode_message(upb_encstate* e, const upb_Message* msg, const upb_MiniTable* m, size_t* size) { size_t pre_len = e->limit - e->ptr; @@ -13767,17 +13677,12 @@ static void encode_message(upb_encstate* e, const upb_Message* msg, size_t ext_count; const upb_Message_Extension* ext = _upb_Message_Getexts(msg, &ext_count); if (ext_count) { - if (e->options & kUpb_EncodeOption_Deterministic) { - _upb_sortedmap sorted; - _upb_mapsorter_pushexts(&e->sorter, ext, ext_count, &sorted); - while (_upb_sortedmap_nextext(&e->sorter, &sorted, &ext)) { - encode_ext(e, ext, m->ext == kUpb_ExtMode_IsMessageSet); - } - _upb_mapsorter_popmap(&e->sorter, &sorted); - } else { - const upb_Message_Extension* end = ext + ext_count; - for (; ext != end; ext++) { - encode_ext(e, ext, m->ext == kUpb_ExtMode_IsMessageSet); + const upb_Message_Extension* end = ext + ext_count; + for (; ext != end; ext++) { + if (UPB_UNLIKELY(m->ext == kUpb_ExtMode_IsMessageSet)) { + encode_msgset_item(e, ext); + } else { + encode_field(e, &ext->data, &ext->ext->sub, &ext->ext->field); } } } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 0af88d642c20..cd703c288fae 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1424,6 +1424,179 @@ UPB_INLINE int upb_Log2CeilingSize(int x) { return 1 << upb_Log2Ceiling(x); } #include +#ifndef UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ +#define UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ + + +// Must be last. + +struct upb_Decoder; +typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, + upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t data); +typedef struct { + uint64_t field_data; + _upb_FieldParser* field_parser; +} _upb_FastTable_Entry; + +typedef enum { + kUpb_ExtMode_NonExtendable = 0, // Non-extendable message. + kUpb_ExtMode_Extendable = 1, // Normal extendable message. + kUpb_ExtMode_IsMessageSet = 2, // MessageSet message. + kUpb_ExtMode_IsMessageSet_ITEM = + 3, // MessageSet item (temporary only, see decode.c) + + // During table building we steal a bit to indicate that the message is a map + // entry. *Only* used during table building! + kUpb_ExtMode_IsMapEntry = 4, +} upb_ExtMode; + +// upb_MiniTable represents the memory layout of a given upb_MessageDef. +// The members are public so generated code can initialize them, +// but users MUST NOT directly read or write any of its members. +struct upb_MiniTable { + const upb_MiniTableSub* subs; + const upb_MiniTableField* fields; + + // Must be aligned to sizeof(void*). Doesn't include internal members like + // unknown fields, extension dict, pointer to msglayout, etc. + uint16_t size; + + uint16_t field_count; + uint8_t ext; // upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 + uint8_t dense_below; + uint8_t table_mask; + uint8_t required_count; // Required fields have the lowest hasbits. + + // To statically initialize the tables of variable length, we need a flexible + // array member, and we need to compile in gnu99 mode (constant initialization + // of flexible array members is a GNU extension, not in C99 unfortunately. + _upb_FastTable_Entry fasttable[]; +}; + +// Map entries aren't actually stored for map fields, they are only used during +// parsing. For parsing, it helps a lot if all map entry messages have the same +// layout. The layout code in mini_table/decode.c will ensure that all map +// entries have this layout. +// +// Note that users can and do create map entries directly, which will also use +// this layout. +// +// NOTE: sync with mini_table/decode.c. +typedef struct { + // We only need 2 hasbits max, but due to alignment we'll use 8 bytes here, + // and the uint64_t helps make this clear. + uint64_t hasbits; + union { + upb_StringView str; // For str/bytes. + upb_value val; // For all other types. + } k; + union { + upb_StringView str; // For str/bytes. + upb_value val; // For all other types. + } v; +} upb_MapEntryData; + +typedef struct { + void* internal_data; + upb_MapEntryData data; +} upb_MapEntry; + +#ifdef __cplusplus +extern "C" { +#endif + +// Computes a bitmask in which the |l->required_count| lowest bits are set, +// except that we skip the lowest bit (because upb never uses hasbit 0). +// +// Sample output: +// requiredmask(1) => 0b10 (0x2) +// requiredmask(5) => 0b111110 (0x3e) +UPB_INLINE uint64_t upb_MiniTable_requiredmask(const upb_MiniTable* l) { + int n = l->required_count; + assert(0 < n && n <= 63); + return ((1ULL << n) - 1) << 1; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ */ + +// Must be last. + +#ifdef __cplusplus +extern "C" { +#endif + +// _upb_mapsorter sorts maps and provides ordered iteration over the entries. +// Since maps can be recursive (map values can be messages which contain other +// maps), _upb_mapsorter can contain a stack of maps. + +typedef struct { + upb_tabent const** entries; + int size; + int cap; +} _upb_mapsorter; + +typedef struct { + int start; + int pos; + int end; +} _upb_sortedmap; + +UPB_INLINE void _upb_mapsorter_init(_upb_mapsorter* s) { + s->entries = NULL; + s->size = 0; + s->cap = 0; +} + +UPB_INLINE void _upb_mapsorter_destroy(_upb_mapsorter* s) { + if (s->entries) free(s->entries); +} + +UPB_INLINE bool _upb_sortedmap_next(_upb_mapsorter* s, const upb_Map* map, + _upb_sortedmap* sorted, upb_MapEntry* ent) { + if (sorted->pos == sorted->end) return false; + const upb_tabent* tabent = s->entries[sorted->pos++]; + upb_StringView key = upb_tabstrview(tabent->key); + _upb_map_fromkey(key, &ent->data.k, map->key_size); + upb_value val = {tabent->val.val}; + _upb_map_fromvalue(val, &ent->data.v, map->val_size); + return true; +} + +UPB_INLINE void _upb_mapsorter_popmap(_upb_mapsorter* s, + _upb_sortedmap* sorted) { + s->size = sorted->start; +} + +bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, + const upb_Map* map, _upb_sortedmap* sorted); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_COLLECTIONS_MAP_SORTER_INTERNAL_H_ */ + +/* +** Our memory representation for parsing tables and messages themselves. +** Functions in this file are used by generated code and possibly reflection. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MESSAGE_INTERNAL_H_ +#define UPB_MESSAGE_INTERNAL_H_ + +#include +#include + + #ifndef UPB_MESSAGE_EXTENSION_INTERNAL_H_ #define UPB_MESSAGE_EXTENSION_INTERNAL_H_ @@ -1696,191 +1869,6 @@ const upb_Message_Extension* _upb_Message_Getext( #endif /* UPB_MESSAGE_EXTENSION_INTERNAL_H_ */ -#ifndef UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ -#define UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ - - -// Must be last. - -struct upb_Decoder; -typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, - upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t data); -typedef struct { - uint64_t field_data; - _upb_FieldParser* field_parser; -} _upb_FastTable_Entry; - -typedef enum { - kUpb_ExtMode_NonExtendable = 0, // Non-extendable message. - kUpb_ExtMode_Extendable = 1, // Normal extendable message. - kUpb_ExtMode_IsMessageSet = 2, // MessageSet message. - kUpb_ExtMode_IsMessageSet_ITEM = - 3, // MessageSet item (temporary only, see decode.c) - - // During table building we steal a bit to indicate that the message is a map - // entry. *Only* used during table building! - kUpb_ExtMode_IsMapEntry = 4, -} upb_ExtMode; - -// upb_MiniTable represents the memory layout of a given upb_MessageDef. -// The members are public so generated code can initialize them, -// but users MUST NOT directly read or write any of its members. -struct upb_MiniTable { - const upb_MiniTableSub* subs; - const upb_MiniTableField* fields; - - // Must be aligned to sizeof(void*). Doesn't include internal members like - // unknown fields, extension dict, pointer to msglayout, etc. - uint16_t size; - - uint16_t field_count; - uint8_t ext; // upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 - uint8_t dense_below; - uint8_t table_mask; - uint8_t required_count; // Required fields have the lowest hasbits. - - // To statically initialize the tables of variable length, we need a flexible - // array member, and we need to compile in gnu99 mode (constant initialization - // of flexible array members is a GNU extension, not in C99 unfortunately. - _upb_FastTable_Entry fasttable[]; -}; - -// Map entries aren't actually stored for map fields, they are only used during -// parsing. For parsing, it helps a lot if all map entry messages have the same -// layout. The layout code in mini_table/decode.c will ensure that all map -// entries have this layout. -// -// Note that users can and do create map entries directly, which will also use -// this layout. -// -// NOTE: sync with mini_table/decode.c. -typedef struct { - // We only need 2 hasbits max, but due to alignment we'll use 8 bytes here, - // and the uint64_t helps make this clear. - uint64_t hasbits; - union { - upb_StringView str; // For str/bytes. - upb_value val; // For all other types. - } k; - union { - upb_StringView str; // For str/bytes. - upb_value val; // For all other types. - } v; -} upb_MapEntryData; - -typedef struct { - void* internal_data; - upb_MapEntryData data; -} upb_MapEntry; - -#ifdef __cplusplus -extern "C" { -#endif - -// Computes a bitmask in which the |l->required_count| lowest bits are set, -// except that we skip the lowest bit (because upb never uses hasbit 0). -// -// Sample output: -// requiredmask(1) => 0b10 (0x2) -// requiredmask(5) => 0b111110 (0x3e) -UPB_INLINE uint64_t upb_MiniTable_requiredmask(const upb_MiniTable* l) { - int n = l->required_count; - assert(0 < n && n <= 63); - return ((1ULL << n) - 1) << 1; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_MINI_TABLE_MESSAGE_INTERNAL_H_ */ - -// Must be last. - -#ifdef __cplusplus -extern "C" { -#endif - -// _upb_mapsorter sorts maps and provides ordered iteration over the entries. -// Since maps can be recursive (map values can be messages which contain other -// maps), _upb_mapsorter can contain a stack of maps. - -typedef struct { - void const** entries; - int size; - int cap; -} _upb_mapsorter; - -typedef struct { - int start; - int pos; - int end; -} _upb_sortedmap; - -UPB_INLINE void _upb_mapsorter_init(_upb_mapsorter* s) { - s->entries = NULL; - s->size = 0; - s->cap = 0; -} - -UPB_INLINE void _upb_mapsorter_destroy(_upb_mapsorter* s) { - if (s->entries) free(s->entries); -} - -UPB_INLINE bool _upb_sortedmap_next(_upb_mapsorter* s, const upb_Map* map, - _upb_sortedmap* sorted, upb_MapEntry* ent) { - if (sorted->pos == sorted->end) return false; - const upb_tabent* tabent = (const upb_tabent*)s->entries[sorted->pos++]; - upb_StringView key = upb_tabstrview(tabent->key); - _upb_map_fromkey(key, &ent->data.k, map->key_size); - upb_value val = {tabent->val.val}; - _upb_map_fromvalue(val, &ent->data.v, map->val_size); - return true; -} - -UPB_INLINE bool _upb_sortedmap_nextext(_upb_mapsorter* s, - _upb_sortedmap* sorted, - const upb_Message_Extension** ext) { - if (sorted->pos == sorted->end) return false; - *ext = (const upb_Message_Extension*)s->entries[sorted->pos++]; - return true; -} - -UPB_INLINE void _upb_mapsorter_popmap(_upb_mapsorter* s, - _upb_sortedmap* sorted) { - s->size = sorted->start; -} - -bool _upb_mapsorter_pushmap(_upb_mapsorter* s, upb_FieldType key_type, - const upb_Map* map, _upb_sortedmap* sorted); - -bool _upb_mapsorter_pushexts(_upb_mapsorter* s, - const upb_Message_Extension* exts, size_t count, - _upb_sortedmap* sorted); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_COLLECTIONS_MAP_SORTER_INTERNAL_H_ */ - -/* -** Our memory representation for parsing tables and messages themselves. -** Functions in this file are used by generated code and possibly reflection. -** -** The definitions in this file are internal to upb. -**/ - -#ifndef UPB_MESSAGE_INTERNAL_H_ -#define UPB_MESSAGE_INTERNAL_H_ - -#include -#include - - #ifndef UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ #define UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ @@ -8813,7 +8801,6 @@ bool upb_FileDef_HasOptions(const upb_FileDef* f); const char* upb_FileDef_Name(const upb_FileDef* f); const UPB_DESC(FileOptions) * upb_FileDef_Options(const upb_FileDef* f); const char* upb_FileDef_Package(const upb_FileDef* f); -const char* upb_FileDef_Edition(const upb_FileDef* f); const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); const upb_FileDef* upb_FileDef_PublicDependency(const upb_FileDef* f, int i); @@ -9926,7 +9913,7 @@ size_t _upb_DefPool_BytesLoaded(const upb_DefPool* s); upb_ExtensionRegistry* _upb_DefPool_ExtReg(const upb_DefPool* s); bool _upb_DefPool_InsertExt(upb_DefPool* s, const upb_MiniTableExtension* ext, - const upb_FieldDef* f); + upb_FieldDef* f); bool _upb_DefPool_InsertSym(upb_DefPool* s, upb_StringView sym, upb_value v, upb_Status* status); bool _upb_DefPool_LookupSym(const upb_DefPool* s, const char* sym, size_t size, @@ -10052,6 +10039,23 @@ UPB_INLINE upb_FileDef* _upb_DefBuilder_File(const upb_DefBuilder* ctx) { void _upb_DefBuilder_CheckIdentSlow(upb_DefBuilder* ctx, upb_StringView name, bool full); +// Verify a relative identifier string. The loop is branchless for speed. +UPB_INLINE void _upb_DefBuilder_CheckIdentNotFull(upb_DefBuilder* ctx, + upb_StringView name) { + bool good = name.size > 0; + + for (size_t i = 0; i < name.size; i++) { + const char c = name.data[i]; + const char d = c | 0x20; // force lowercase + const bool is_alpha = (('a' <= d) & (d <= 'z')) | (c == '_'); + const bool is_numer = ('0' <= c) & (c <= '9') & (i != 0); + + good &= is_alpha | is_numer; + } + + if (!good) _upb_DefBuilder_CheckIdentSlow(ctx, name, false); +} + // Verify a full identifier string. This is slightly more complicated than // verifying a relative identifier string because we must track '.' chars. UPB_INLINE void _upb_DefBuilder_CheckIdentFull(upb_DefBuilder* ctx, @@ -10155,14 +10159,6 @@ int _upb_FieldDef_LayoutIndex(const upb_FieldDef* f); uint64_t _upb_FieldDef_Modifiers(const upb_FieldDef* f); void _upb_FieldDef_Resolve(upb_DefBuilder* ctx, const char* prefix, upb_FieldDef* f); -void _upb_FieldDef_BuildMiniTableExtension(upb_DefBuilder* ctx, - const upb_FieldDef* f); - -// Allocate and initialize an array of |n| extensions (field defs). -upb_FieldDef* _upb_Extensions_New( - upb_DefBuilder* ctx, int n, - const UPB_DESC(FieldDescriptorProto) * const* protos, const char* prefix, - upb_MessageDef* m); // Allocate and initialize an array of |n| field defs. upb_FieldDef* _upb_FieldDefs_New( @@ -10227,7 +10223,6 @@ bool _upb_MessageDef_Insert(upb_MessageDef* m, const char* name, size_t size, void _upb_MessageDef_InsertField(upb_DefBuilder* ctx, upb_MessageDef* m, const upb_FieldDef* f); bool _upb_MessageDef_IsValidExtensionNumber(const upb_MessageDef* m, int n); -void _upb_MessageDef_CreateMiniTable(upb_DefBuilder* ctx, upb_MessageDef* m); void _upb_MessageDef_LinkMiniTable(upb_DefBuilder* ctx, const upb_MessageDef* m); void _upb_MessageDef_Resolve(upb_DefBuilder* ctx, upb_MessageDef* m); From 649cc30872f56da00f18d20ca8f85859423a6fb3 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 16 Feb 2023 14:18:17 -0800 Subject: [PATCH 054/463] Open-source some of our string_view tests. This support was added in 22.x, but we don't have any tests locking it down. Closes https://github.com/protocolbuffers/protobuf/issues/8096 PiperOrigin-RevId: 510242380 --- .github/workflows/test_runner.yml | 8 ++--- CMakeLists.txt | 2 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/README.md | 6 ++-- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 2 +- java/util/pom.xml | 2 +- php/ext/google/protobuf/protobuf.h | 2 +- protobuf_version.bzl | 10 +++--- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 +-- src/google/protobuf/compiler/plugin.pb.h | 4 +-- src/google/protobuf/descriptor.pb.h | 4 +-- src/google/protobuf/message_unittest.inc | 34 +++++++++++++++++++ src/google/protobuf/port_def.inc | 8 ++--- src/google/protobuf/stubs/common.h | 8 ++--- 26 files changed, 77 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index 9fa37f185f64..ce3268a21249 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -21,8 +21,8 @@ on: branches: - main - '[0-9]+.x' - # The 21.x branch still uses Kokoro - - '!21.x' + # The 21.x and 22.x branches still use Kokoro + - '!2[12].x' # For testing purposes so we can stage this on the `gha` branch. - gha @@ -31,8 +31,8 @@ on: branches: - main - '[0-9]+.x' - # The 21.x branch still uses Kokoro - - '!21.x' + # The 21.x and 22.x branches still use Kokoro + - '!2[12].x' # For testing purposes so we can stage this on the `gha` branch. - gha diff --git a/CMakeLists.txt b/CMakeLists.txt index c71a74d4a10c..20786bd3ff4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,7 @@ if (protobuf_BUILD_SHARED_LIBS) endif () # Version metadata -set(protobuf_VERSION_STRING "4.22.0") +set(protobuf_VERSION_STRING "3.21.4") set(protobuf_DESCRIPTION "Protocol Buffers") set(protobuf_CONTACT "protobuf@googlegroups.com") diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 8b90ccce49a5..4999537e3ef6 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '4.22.0' + s.version = '3.21.12' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index 286db4b45c71..b97dfea239eb 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.22.0' + s.version = '3.21.12' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index e8bc49731bb1..f4bf457567e7 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools

Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.22.0 + 3.21.12 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 026a1d718dc3..35f252db7ffe 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -5,7 +5,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.22.0 + 3.21.12 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/java/README.md b/java/README.md index 8f60bb2af3f2..85039148769e 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.22.0 + 3.21.12 ``` @@ -37,14 +37,14 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.22.0 + 3.21.12 ``` ### Gradle If you are using Gradle, add the following to your `build.gradle` file's -dependencies: `implementation 'com.google.protobuf:protobuf-java:3.22.0'` +dependencies: `implementation 'com.google.protobuf:protobuf-java:3.21.12'` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index f792a629664b..c01964c85eec 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.22.0 + 3.21.12 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 6f32c9d59fd0..83b7351f7ec9 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 protobuf-java diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index c91bd889adcf..50dcc8340b34 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 52a368d58f86..97b7f70c6ba0 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index 035b86c2bfe4..ad920792f3f9 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.22.0 + 3.21.12 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 716ff74d324f..af34943dabf7 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index cedeb04d0ea0..c0bdaeae6b18 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index 4ffc34dde8f0..9b61b9826a7e 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.22.0 + 3.21.6 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index 84cba14001e8..c13df651c8e0 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.0 + 3.21.12 protobuf-java-util diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index e26ee44285a6..efbd45eac052 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.22.0" +#define PHP_PROTOBUF_VERSION "3.21.12" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protobuf_version.bzl b/protobuf_version.bzl index e3e45eba5043..ec39a4e12371 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,5 +1,5 @@ -PROTOC_VERSION = "22.0" -PROTOBUF_JAVA_VERSION = "3.22.0" -PROTOBUF_PYTHON_VERSION = "4.22.0" -PROTOBUF_PHP_VERSION = "3.22.0" -PROTOBUF_RUBY_VERSION = "3.22.0" +PROTOC_VERSION = "21.12" +PROTOBUF_JAVA_VERSION = "3.21.12" +PROTOBUF_PYTHON_VERSION = "4.21.12" +PROTOBUF_PHP_VERSION = "3.21.12" +PROTOBUF_RUBY_VERSION = "3.21.12" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index a301349e31e6..e7555ee1010f 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.22.0' +__version__ = '4.21.12' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 174657dee340..a96b5e9cacd4 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.22.0" + s.version = "3.21.12" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index 1d3de9b5634b..68dadacfb0a7 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.22.0 + 3.21.12 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.22.0 + 3.21.12 org.jruby diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index d5480b1aae3c..d3661160280e 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index d2005d163d24..4753a9de8261 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 4c8ffdcb500d..4305f5310c8c 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -515,6 +515,8 @@ TEST(MESSAGE_TEST_NAME, ParseStrictlyBoundedStream) { TEST(MESSAGE_TEST_NAME, AllSetMethodsOnStringField) { UNITTEST::TestAllTypes msg; + msg.set_optional_string(absl::string_view("Abcdef")); + EXPECT_EQ(msg.optional_string(), "Abcdef"); msg.set_optional_string("Asciiz"); EXPECT_EQ(msg.optional_string(), "Asciiz"); @@ -535,6 +537,38 @@ TEST(MESSAGE_TEST_NAME, AllSetMethodsOnStringField) { EXPECT_EQ(msg.optional_string(), "std::string value 3"); } + +TEST(MESSAGE_TEST_NAME, AllAddMethodsOnRepeatedStringField) { + UNITTEST::TestAllTypes msg; + + msg.add_repeated_string(absl::string_view("Abcdef")); + EXPECT_EQ(msg.repeated_string(0), "Abcdef"); + msg.clear_repeated_string(); + + msg.add_repeated_string("Asciiz"); + EXPECT_EQ(msg.repeated_string(0), "Asciiz"); + msg.clear_repeated_string(); + + msg.add_repeated_string("Length delimited", 6); + EXPECT_EQ(msg.repeated_string(0), "Length"); + msg.clear_repeated_string(); + + std::string value = "std::string value 1"; + msg.add_repeated_string(value); + EXPECT_EQ(msg.repeated_string(0), "std::string value 1"); + msg.clear_repeated_string(); + + value = "std::string value 2"; + msg.add_repeated_string(std::cref(value)); + EXPECT_EQ(msg.repeated_string(0), "std::string value 2"); + msg.clear_repeated_string(); + + value = "std::string value 3"; + msg.add_repeated_string(std::move(value)); + EXPECT_EQ(msg.repeated_string(0), "std::string value 3"); + msg.clear_repeated_string(); +} + TEST(MESSAGE_TEST_NAME, SuccessAfterParsingFailure) { UNITTEST::NestedTestAllTypes o, p, q; constexpr int kDepth = 5; diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 35df77ae674a..3283030c16dd 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -218,17 +218,17 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 4022000 +#define PROTOBUF_VERSION 3021012 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined #endif -#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 4022000 +#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3021000 #ifdef PROTOBUF_MIN_PROTOC_VERSION #error PROTOBUF_MIN_PROTOC_VERSION was previously defined #endif -#define PROTOBUF_MIN_PROTOC_VERSION 4022000 +#define PROTOBUF_MIN_PROTOC_VERSION 3021000 #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined @@ -433,7 +433,7 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // The minimum library version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 4022000 +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3021000 #ifdef PROTOBUF_RTTI #error PROTOBUF_RTTI was previously defined diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 5f1652d25d10..a3b0c6918049 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -66,7 +66,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 4022000 +#define GOOGLE_PROTOBUF_VERSION 3021012 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" @@ -74,15 +74,15 @@ namespace internal { // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 4022000; +static const int kMinHeaderVersionForLibrary = 3021000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 4022000 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3021000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 4022000; +static const int kMinHeaderVersionForProtoc = 3021000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. From d41deb9dc6602701b157739a314ad6405f9b89ed Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 16 Feb 2023 15:10:54 -0800 Subject: [PATCH 055/463] Internal change that helps identify the origin of reflection calls. PiperOrigin-RevId: 510255963 --- pkg/BUILD.bazel | 1 + src/google/protobuf/BUILD.bazel | 25 +++ src/google/protobuf/reflection_mode.cc | 57 +++++++ src/google/protobuf/reflection_mode.h | 165 ++++++++++++++++++++ src/google/protobuf/reflection_mode_test.cc | 115 ++++++++++++++ 5 files changed, 363 insertions(+) create mode 100644 src/google/protobuf/reflection_mode.cc create mode 100644 src/google/protobuf/reflection_mode.h create mode 100644 src/google/protobuf/reflection_mode_test.cc diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index a3a11399ad51..25513f5c088a 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -362,6 +362,7 @@ cc_dist_library( "//src/google/protobuf:arena_align", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf:wkt_cc_proto", + "//src/google/protobuf:reflection_mode", "//src/google/protobuf/compiler:importer", "//src/google/protobuf/json", "//src/google/protobuf/util:delimited_message_util", diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index e3db90cc513e..90dbb7b33a73 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -1314,6 +1314,31 @@ cc_test( ], ) +cc_library( + name = "reflection_mode", + srcs = ["reflection_mode.cc"], + hdrs = ["reflection_mode.h"], + include_prefix = "google/protobuf", + visibility = [ + "//:__subpackages__", + "//src/google/protobuf:__subpackages__", + ], + deps = [ + ":port_def", + ":protobuf", + ], +) + +cc_test( + name = "reflection_mode_test", + srcs = ["reflection_mode_test.cc"], + deps = [ + ":reflection_mode", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + ################################################################################ # Helper targets for Kotlin tests ################################################################################ diff --git a/src/google/protobuf/reflection_mode.cc b/src/google/protobuf/reflection_mode.cc new file mode 100644 index 000000000000..7611c7b69cdd --- /dev/null +++ b/src/google/protobuf/reflection_mode.cc @@ -0,0 +1,57 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/reflection_mode.h" + +// Must be included last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace internal { + +#if !defined(PROTOBUF_NO_THREADLOCAL) + +#if defined(PROTOBUF_USE_DLLS) +ReflectionMode& ScopedReflectionMode::reflection_mode() { + static PROTOBUF_THREAD_LOCAL ReflectionMode reflection_mode = + ReflectionMode::kDefault; + return reflection_mode; +} +#else +PROTOBUF_CONSTINIT PROTOBUF_THREAD_LOCAL ReflectionMode + ScopedReflectionMode::reflection_mode_ = ReflectionMode::kDefault; +#endif + +#endif + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/reflection_mode.h b/src/google/protobuf/reflection_mode.h new file mode 100644 index 000000000000..e9ff3408f0cf --- /dev/null +++ b/src/google/protobuf/reflection_mode.h @@ -0,0 +1,165 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This header provides support for a per thread 'reflection mode'. +// +// Some protocol buffer optimizations use interceptors to determine which +// fields are effectively used in the application. These optimizations are +// disabled if certain reflection calls are intercepted as the assumption is +// then that any field data can be accessed. +// +// The 'reflection mode' defined in this header is intended to be used by +// logic such as ad-hoc profilers to indicate that any scoped reflection usage +// is not originating from, or affecting application code. This reflection mode +// can then be used by such interceptors to ignore any reflection calls not +// affecting the application behavior. + +#ifndef GOOGLE_PROTOBUF_REFLECTION_MODE_H__ +#define GOOGLE_PROTOBUF_REFLECTION_MODE_H__ + +#include + +// Must be included last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace internal { + +// The ReflectionModes are ordered in observability levels: +// kDefault: Lowest level. All reflection calls are observable. +// kDebugString: Middle level. Only reflection calls in Message::DebugString are +// observable. +// kDiagnostics: Highest level. No reflection calls are observable. +enum class ReflectionMode { + kDefault, + kDebugString, + kDiagnostics, +}; + +// Returns the current ReflectionMode of protobuf for the current thread. This +// reflection mode can be used by interceptors to ignore any reflection calls +// not affecting the application behavior. +// Always returns `kDefault' if the current platform does not support thread +// local data. +ReflectionMode GetReflectionMode(); + +// Scoping class to set the specific ReflectionMode for a given scope. +class PROTOBUF_EXPORT ScopedReflectionMode final { + public: + // Sets the current reflection mode, which will be restored at destruction. + // The reflection mode can only be 'elevated' in observability levels. + // For instance, if the current mode is `kDiagnostics` then scope will remain + // unchanged regardless of `mode`. + explicit ScopedReflectionMode(ReflectionMode mode); + + // Restores the previous reflection mode. + ~ScopedReflectionMode(); + + // Returns the scoped ReflectionMode for the current thread. + // See `GetReflectionMode()` for more information on purpose and usage. + static ReflectionMode current_reflection_mode(); + + // ScopedReflectionMode is only intended to be used as a locally scoped + // instance to set a reflection mode for the code scoped by this instance. + ScopedReflectionMode(const ScopedReflectionMode&) = delete; + ScopedReflectionMode& operator=(const ScopedReflectionMode&) = delete; + + private: +#if !defined(PROTOBUF_NO_THREADLOCAL) + const ReflectionMode previous_mode_; +#if defined(PROTOBUF_USE_DLLS) + static ReflectionMode& reflection_mode(); +#else + PROTOBUF_CONSTINIT static PROTOBUF_THREAD_LOCAL ReflectionMode + reflection_mode_; +#endif // PROTOBUF_USE_DLLS +#endif // !PROTOBUF_NO_THREADLOCAL +}; + +#if !defined(PROTOBUF_NO_THREADLOCAL) + +#if defined(PROTOBUF_USE_DLLS) + +inline ScopedReflectionMode::ScopedReflectionMode(ReflectionMode mode) + : previous_mode_(reflection_mode()) { + if (mode > reflection_mode()) { + reflection_mode() = mode; + } +} + +inline ScopedReflectionMode::~ScopedReflectionMode() { + reflection_mode() = previous_mode_; +} + +inline ReflectionMode ScopedReflectionMode::current_reflection_mode() { + return reflection_mode(); +} + +#else + +inline ScopedReflectionMode::ScopedReflectionMode(ReflectionMode mode) + : previous_mode_(reflection_mode_) { + if (mode > reflection_mode_) { + reflection_mode_ = mode; + } +} + +inline ScopedReflectionMode::~ScopedReflectionMode() { + reflection_mode_ = previous_mode_; +} + +inline ReflectionMode ScopedReflectionMode::current_reflection_mode() { + return reflection_mode_; +} + +#endif // PROTOBUF_USE_DLLS + +#else + +inline ScopedReflectionMode::ScopedReflectionMode(ReflectionMode mode) {} +inline ScopedReflectionMode::~ScopedReflectionMode() {} +inline ReflectionMode ScopedReflectionMode::current_reflection_mode() { + return ReflectionMode::kDefault; +} + +#endif // !PROTOBUF_NO_THREADLOCAL + +inline ReflectionMode GetReflectionMode() { + return ScopedReflectionMode::current_reflection_mode(); +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" + +#endif // GOOGLE_PROTOBUF_REFLECTION_MODE_H__ diff --git a/src/google/protobuf/reflection_mode_test.cc b/src/google/protobuf/reflection_mode_test.cc new file mode 100644 index 000000000000..9320888add62 --- /dev/null +++ b/src/google/protobuf/reflection_mode_test.cc @@ -0,0 +1,115 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "google/protobuf/reflection_mode.h" + +#include +#include + +namespace google { +namespace protobuf { +namespace internal { + +#ifndef PROTOBUF_NO_THREADLOCAL + +TEST(ReflectionModeTest, SimpleScopedReflection) { + ASSERT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + ScopedReflectionMode scope(ReflectionMode::kDiagnostics); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDiagnostics); +} + +TEST(ReflectionModeTest, CleanNestedScopedReflection) { + ASSERT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + { + ScopedReflectionMode scope1(ReflectionMode::kDebugString); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDebugString); + { + ScopedReflectionMode scope2(ReflectionMode::kDiagnostics); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDiagnostics); + } + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDebugString); + } + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); +} + +TEST(ReflectionModeTest, UglyNestedScopedReflection) { + ASSERT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + ScopedReflectionMode scope1(ReflectionMode::kDebugString); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDebugString); + ScopedReflectionMode scope2(ReflectionMode::kDiagnostics); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDiagnostics); +} + +TEST(ReflectionModeTest, DebugStringModeDoesNotReplaceDiagnosticsMode) { + ASSERT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + ScopedReflectionMode scope1(ReflectionMode::kDiagnostics); + { + ScopedReflectionMode scope2(ReflectionMode::kDebugString); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDiagnostics); + } +} + +#else + +TEST(ReflectionModeTest, AlwaysReturnDefaultWhenNoThreadLocal) { + ASSERT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + { + ScopedReflectionMode scope1(ReflectionMode::kDebugString); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + { + ScopedReflectionMode scope2(ReflectionMode::kDiagnostics); + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + } + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); + } + EXPECT_EQ(ScopedReflectionMode::current_reflection_mode(), + ReflectionMode::kDefault); +} + +#endif + +} // namespace internal +} // namespace protobuf +} // namespace google From 6d776dfed52e36a58f318c0ba3acc93445ed710e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 01:21:17 -0800 Subject: [PATCH 056/463] protobuf: fix a data race in AbstractProtobufList Unconditional update of isMutable causes data races for LazyStringArrayList::EMPTY_LIST that is used as the default value for protobuf fields. Not writing to it unconditionally may also improve performance by reducing contention for the presumably widely shared EMPTY_LIST object. PiperOrigin-RevId: 510366298 --- .../main/java/com/google/protobuf/AbstractProtobufList.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java b/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java index e792d7d98119..b8baaa083e84 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java @@ -130,7 +130,9 @@ public boolean isModifiable() { @Override public final void makeImmutable() { - isMutable = false; + if (isMutable) { + isMutable = false; + } } @Override From 4f6ef7e4d88a74dfcd82b36ef46844b22b9e54b1 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Fri, 17 Feb 2023 04:34:54 -0800 Subject: [PATCH 057/463] internal change PiperOrigin-RevId: 510398988 --- src/google/protobuf/compiler/rust/METADATA | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/google/protobuf/compiler/rust/METADATA diff --git a/src/google/protobuf/compiler/rust/METADATA b/src/google/protobuf/compiler/rust/METADATA new file mode 100644 index 000000000000..9b19d6cdbd8b --- /dev/null +++ b/src/google/protobuf/compiler/rust/METADATA @@ -0,0 +1,5 @@ +# go/google3metadata +# proto-file: devtools/metadata/metadata.proto +# proto-message: MetaData + +presubmit: { review_notify: "protobuf-rust+reviews" } From bab55c25531ee407ddd6c065927f96b34c337ae6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 06:22:45 -0800 Subject: [PATCH 058/463] [ObjC] Inline the logic of parseDelimited... The last attempted (rolled back) resulted in changes in visible behavior: - It consumed a recursion depth on the input stream (wrong) - It resulted in different error codes for some malformed/truncated data. PiperOrigin-RevId: 510415215 --- objectivec/GPBMessage.m | 51 ++++++++++++-------------- objectivec/GPBMessage_PackagePrivate.h | 5 --- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index cb8d70a47573..b29540a30fc5 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -28,6 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#import #import "GPBMessage_PackagePrivate.h" #import @@ -2000,19 +2001,6 @@ - (void)mergeFromData:(NSData *)data extensionRegistry:(id [input release]; } -#pragma mark - mergeDelimitedFrom - -- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input - extensionRegistry:(id)extensionRegistry { - GPBCodedInputStreamState *state = &input->state_; - if (GPBCodedInputStreamIsAtEnd(state)) { - return; - } - NSData *data = GPBCodedInputStreamReadRetainedBytesNoCopy(state); - [self mergeFromData:data extensionRegistry:extensionRegistry]; - [data release]; -} - #pragma mark - Parse From Data Support + (instancetype)parseFromData:(NSData *)data error:(NSError **)errorPtr { @@ -2039,27 +2027,36 @@ + (instancetype)parseFromCodedInputStream:(GPBCodedInputStream *)input + (instancetype)parseDelimitedFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { - GPBMessage *message = [[[self alloc] init] autorelease]; + GPBCodedInputStreamState *state = &input->state_; + // This doesn't completely match the C++, but if the stream has nothing, just make an empty + // message. + if (GPBCodedInputStreamIsAtEnd(state)) { + return [[[self alloc] init] autorelease]; + } + + // Manually extract the data and parse it. If we read a varint and push a limit, that consumes + // some of the recursion buffer which isn't correct, it also can result in a change in error + // codes for attempts to parse partial data; and there are projects sensitive to that, so this + // maintains existing error flows. + + // Extract the data, but in a "no copy" mode since we will immediately parse it so this NSData + // is transient. + NSData *data = nil; @try { - [message mergeDelimitedFromCodedInputStream:input extensionRegistry:extensionRegistry]; - if (errorPtr) { - *errorPtr = nil; - } + data = GPBCodedInputStreamReadRetainedBytesNoCopy(state); } @catch (NSException *exception) { - message = nil; if (errorPtr) { *errorPtr = ErrorFromException(exception); } + return nil; } -#ifdef DEBUG - if (message && !message.initialized) { - message = nil; - if (errorPtr) { - *errorPtr = MessageError(GPBMessageErrorCodeMissingRequiredField, nil); - } + + GPBMessage *result = [self parseFromData:data extensionRegistry:extensionRegistry error:errorPtr]; + [data release]; + if (result && errorPtr) { + *errorPtr = nil; } -#endif - return message; + return result; } #pragma mark - Unknown Field Support diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 70b47a59bc88..1b3a99332df8 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -80,11 +80,6 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input extensionRegistry:(id)extensionRegistry; -// Parses the next delimited message of this type from the input and merges it -// with this message. -- (void)mergeDelimitedFromCodedInputStream:(GPBCodedInputStream *)input - extensionRegistry:(id)extensionRegistry; - - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data; @end From 32aaf9eeb53f6afb146da2acac360fb5eb3d6db3 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 06:35:02 -0800 Subject: [PATCH 059/463] [ObjC] Ensure objects aren't leaked on parsing failures. Add the objects to the object graph and balance the retain count before doing the parsing. This ensure if an error is hit (and a throw happens), the object(s) won't be leaked. Parsing will always mutate the graph, so yes this includes more mutations in failure cases, but other fields could always be modified before the bad data is encountered. But even then, that edge case *only* apples to api users that are explicitly *merge* (-mergeFrom...), the majority of the calls are to +parseFromData:error:, so the entire graph is released on failure. PiperOrigin-RevId: 510417377 --- objectivec/GPBMessage.m | 21 ++++++++++++++------- objectivec/GPBUnknownFieldSet.m | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index b29540a30fc5..0bdf4bb8a777 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -1996,9 +1996,12 @@ - (void)clearExtension:(GPBExtensionDescriptor *)extension { - (void)mergeFromData:(NSData *)data extensionRegistry:(id)extensionRegistry { GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; - [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; - [input checkLastTagWas:0]; - [input release]; + @try { + [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; + [input checkLastTagWas:0]; + } @finally { + [input release]; + } } #pragma mark - Parse From Data Support @@ -2204,8 +2207,8 @@ static void MergeSingleFieldFromCodedInputStream(GPBMessage *self, GPBFieldDescr [input readMessage:message extensionRegistry:extensionRegistry]; } else { GPBMessage *message = [[field.msgClass alloc] init]; - [input readMessage:message extensionRegistry:extensionRegistry]; GPBSetRetainedObjectIvarWithFieldPrivate(self, field, message); + [input readMessage:message extensionRegistry:extensionRegistry]; } break; } @@ -2218,8 +2221,8 @@ static void MergeSingleFieldFromCodedInputStream(GPBMessage *self, GPBFieldDescr [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; } else { GPBMessage *message = [[field.msgClass alloc] init]; - [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; GPBSetRetainedObjectIvarWithFieldPrivate(self, field, message); + [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; } break; } @@ -2327,16 +2330,20 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( #undef CASE_NOT_PACKED_OBJECT case GPBDataTypeMessage: { GPBMessage *message = [[field.msgClass alloc] init]; - [input readMessage:message extensionRegistry:extensionRegistry]; [(NSMutableArray *)genericArray addObject:message]; + // The array will now retain message, so go ahead and release it in case + // -readMessage:extensionRegistry: throws so it won't be leaked. [message release]; + [input readMessage:message extensionRegistry:extensionRegistry]; break; } case GPBDataTypeGroup: { GPBMessage *message = [[field.msgClass alloc] init]; - [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; [(NSMutableArray *)genericArray addObject:message]; + // The array will now retain message, so go ahead and release it in case + // -readGroup:extensionRegistry: throws so it won't be leaked. [message release]; + [input readGroup:GPBFieldNumber(field) message:message extensionRegistry:extensionRegistry]; break; } case GPBDataTypeEnum: { diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index 1dce73893405..fd3693548005 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -318,10 +318,12 @@ - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input { } case GPBWireFormatStartGroup: { GPBUnknownFieldSet *unknownFieldSet = [[GPBUnknownFieldSet alloc] init]; - [input readUnknownGroup:number message:unknownFieldSet]; GPBUnknownField *field = [self mutableFieldForNumber:number create:YES]; [field addGroup:unknownFieldSet]; + // The field will now retain unknownFieldSet, so go ahead and release it in case + // -readUnknownGroup:message: throws so it won't be leaked. [unknownFieldSet release]; + [input readUnknownGroup:number message:unknownFieldSet]; return YES; } case GPBWireFormatEndGroup: From b7e0f842e9176efb297c1cd7ff74a6bd70004732 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 17 Feb 2023 08:59:41 -0800 Subject: [PATCH 060/463] Remove recursive checkout for purely Bazel tests This will prevent unnecessary network access. PiperOrigin-RevId: 510444213 --- .github/workflows/test_python.yml | 1 - .github/workflows/test_ruby.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index e7b76a420956..04fa54acc0a5 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -65,7 +65,6 @@ jobs: - name: Checkout pending changes uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: - submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Pin Python version diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index e22bf8a3568d..f5e3663fd2b2 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -83,7 +83,6 @@ jobs: - name: Checkout pending changes uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: - submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Pin Ruby version From 15d250507061338987e05b4778e9f61e96458312 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 17 Feb 2023 09:41:11 -0800 Subject: [PATCH 061/463] Fix staleness flakes from race condition. If a change is committed while we're regenerating stale files, we should just bail and let the next job handle the refresh. PiperOrigin-RevId: 510453613 --- .github/workflows/staleness_check.yml | 1 + .github/workflows/staleness_refresh.yml | 3 ++- push_auto_update.sh => ci/push_auto_update.sh | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) rename push_auto_update.sh => ci/push_auto_update.sh (94%) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 022d64eb3905..63c4cf39ce1f 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -4,6 +4,7 @@ on: schedule: # Run daily at 10 AM UTC (2 AM PDT) - cron: 0 10 * * * + workflow_dispatch: permissions: {} jobs: diff --git a/.github/workflows/staleness_refresh.yml b/.github/workflows/staleness_refresh.yml index 8f083fdc6b9b..c53e4ed9fb80 100644 --- a/.github/workflows/staleness_refresh.yml +++ b/.github/workflows/staleness_refresh.yml @@ -8,6 +8,7 @@ on: # The 21.x branch predates support for auto-generation, so we make sure # to exclude it. - '!21.x' + workflow_dispatch: permissions: {} jobs: @@ -31,4 +32,4 @@ jobs: - name: Configure name and email address in Git run: cd ${{ github.workspace }} && git config user.name "Protobuf Team Bot" && git config user.email "protobuf-team-bot@google.com" - name: Commit and push update - run: cd ${{ github.workspace }} && ./push_auto_update.sh + run: cd ${{ github.workspace }} && ./ci/push_auto_update.sh diff --git a/push_auto_update.sh b/ci/push_auto_update.sh similarity index 94% rename from push_auto_update.sh rename to ci/push_auto_update.sh index 9972da6acf51..ea96cbec33e0 100755 --- a/push_auto_update.sh +++ b/ci/push_auto_update.sh @@ -8,7 +8,7 @@ set -ex # Cd to the repo root. -cd $(dirname -- "$0") +cd $(dirname -- "$0")/.. previous_commit_title=$(git log -1 --pretty='%s') @@ -39,7 +39,6 @@ else commit_message="Auto-generate files" fi -git pull --rebase git add -A git diff --staged --quiet || git commit -am "$commit_message" -git push +git push || echo "Conflicting commit hit, retrying in next job..." From dc952c0193ff5aec778e7cc6e325580ad70c62b5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 17:52:56 +0000 Subject: [PATCH 062/463] Auto-generate files after cl/510453613 --- src/file_lists.cmake | 3 +++ src/google/protobuf/any.pb.h | 4 ++-- src/google/protobuf/api.pb.h | 4 ++-- src/google/protobuf/duration.pb.h | 4 ++-- src/google/protobuf/empty.pb.h | 4 ++-- src/google/protobuf/field_mask.pb.h | 4 ++-- src/google/protobuf/source_context.pb.h | 4 ++-- src/google/protobuf/struct.pb.h | 4 ++-- src/google/protobuf/timestamp.pb.h | 4 ++-- src/google/protobuf/type.pb.h | 4 ++-- src/google/protobuf/wrappers.pb.h | 4 ++-- 11 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index d7542479980d..ba9ad86e0b05 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -68,6 +68,7 @@ set(libprotobuf_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/message_lite.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/parse_context.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_mode.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_ops.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_ptr_field.cc @@ -165,6 +166,7 @@ set(libprotobuf_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/port_undef.inc ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection.h ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_internal.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_mode.h ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_ops.h ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_ptr_field.h @@ -598,6 +600,7 @@ set(protobuf_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/proto3_arena_lite_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/proto3_arena_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/proto3_lite_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_mode_test.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_ops_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field_reflection_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/repeated_field_unittest.cc diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 6bedd8fae78c..493e9f439119 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 4e7097ac41f5..98332c726ece 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 69f4ada05fdd..35ea4f07b061 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 7fa63da62e33..b4d58cf6577d 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index bf9954e71cfc..7f835b317cb5 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 45bc094f61de..dced2d3fc6c7 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 5ff466f41e50..f88800f3100d 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index b7b25cf43bdf..3c0b4698807c 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 710f8cf7b83a..c89a53a356b5 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index d5fb290c8f77..ac85c6ff00c5 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 4022000 +#if PROTOBUF_VERSION < 3021000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." From e1e30cebba851c2401b1bc9a456a5975c66ac3af Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Fri, 17 Feb 2023 09:52:36 -0800 Subject: [PATCH 063/463] Emplace StringBlock entries on the arena block as long as there is space available. PiperOrigin-RevId: 510456100 --- src/google/protobuf/arena.cc | 44 +++++------ src/google/protobuf/arena.h | 5 ++ .../protobuf/repeated_field_unittest.cc | 48 +++++++----- src/google/protobuf/serial_arena.h | 13 ++++ src/google/protobuf/string_block.h | 49 +++++++++++-- src/google/protobuf/string_block_test.cc | 73 ++++++++++++++++++- 6 files changed, 183 insertions(+), 49 deletions(-) diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 071360f14eb0..5d25d65e1f39 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -197,18 +197,21 @@ void* SerialArena::AllocateAlignedFallback(size_t n) { PROTOBUF_NOINLINE void* SerialArena::AllocateFromStringBlockFallback() { + ABSL_DCHECK_EQ(string_block_unused_.load(std::memory_order_relaxed), 0U); if (string_block_) { - ABSL_DCHECK_EQ(string_block_unused_.load(std::memory_order_relaxed), 0U); - space_used_.store(space_used_.load(std::memory_order_relaxed) + - string_block_->effective_size(), - std::memory_order_relaxed); + AddSpaceUsed(string_block_->effective_size()); } - string_block_ = StringBlock::New(string_block_); - space_allocated_.store(space_allocated_.load(std::memory_order_relaxed) + - string_block_->allocated_size(), - std::memory_order_relaxed); - + void* ptr; + size_t size = StringBlock::NextSize(string_block_); + if (MaybeAllocateAligned(size, &ptr)) { + // Correct space_used_ to avoid double counting + AddSpaceUsed(-size); + string_block_ = StringBlock::Emplace(ptr, size, string_block_); + } else { + string_block_ = StringBlock::New(string_block_); + AddSpaceAllocated(string_block_->allocated_size()); + } size_t unused = string_block_->effective_size() - sizeof(std::string); string_block_unused_.store(unused, std::memory_order_relaxed); return string_block_->AtOffset(unused); @@ -240,8 +243,7 @@ void SerialArena::AllocateNewBlock(size_t n) { // Record how much used in this block. used = static_cast(ptr() - old_head->Pointer(kBlockHeaderSize)); wasted = old_head->size - used; - space_used_.store(space_used_.load(std::memory_order_relaxed) + used, - std::memory_order_relaxed); + AddSpaceUsed(used); } // TODO(sbenza): Evaluate if pushing unused space into the cached blocks is a @@ -253,9 +255,7 @@ void SerialArena::AllocateNewBlock(size_t n) { // We don't want to emit an expensive RMW instruction that requires // exclusive access to a cacheline. Hence we write it in terms of a // regular add. - space_allocated_.store( - space_allocated_.load(std::memory_order_relaxed) + mem.n, - std::memory_order_relaxed); + AddSpaceAllocated(mem.n); ThreadSafeArenaStats::RecordAllocateStats(parent_.arena_stats_.MutableStats(), /*used=*/used, /*allocated=*/mem.n, wasted); @@ -278,19 +278,21 @@ uint64_t SerialArena::SpaceUsed() const { // usage of the *current* block. // TODO(mkruskal) Consider eliminating this race in exchange for a possible // performance hit on ARM (see cl/455186837). - uint64_t current_space_used = - string_block_ ? string_block_->effective_size() - - string_block_unused_.load(std::memory_order_relaxed) - : 0; + + uint64_t space_used = 0; + if (string_block_) { + size_t unused = string_block_unused_.load(std::memory_order_relaxed); + space_used += string_block_->effective_size() - unused; + } const ArenaBlock* h = head_.load(std::memory_order_acquire); - if (h->IsSentry()) return current_space_used; + if (h->IsSentry()) return space_used; const uint64_t current_block_size = h->size; - current_space_used += std::min( + space_used += std::min( static_cast( ptr() - const_cast(h)->Pointer(kBlockHeaderSize)), current_block_size); - return current_space_used + space_used_.load(std::memory_order_relaxed); + return space_used + space_used_.load(std::memory_order_relaxed); } size_t SerialArena::FreeStringBlocks(StringBlock* string_block, diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 23e6999c4c0c..e9e7a15d4e4d 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -681,6 +681,11 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { friend struct internal::ArenaTestPeer; }; +template <> +inline void* Arena::AllocateInternal() { + return impl_.AllocateFromStringBlock(); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index 248f01e6be18..f3b4f23b4bf4 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -201,34 +201,24 @@ TEST(RepeatedField, ArenaAllocationSizesMatchExpectedValues) { CheckAllocationSizes>(false); } -template -void CheckNaturalGrowthOnArenasReuseBlocks(bool is_ptr) { +TEST(RepeatedField, NaturalGrowthOnArenasReuseBlocks) { Arena arena; - std::vector values; - using T = typename Rep::value_type; + std::vector*> values; static constexpr int kNumFields = 100; static constexpr int kNumElems = 1000; for (int i = 0; i < kNumFields; ++i) { - values.push_back(Arena::CreateMessage(&arena)); + values.push_back(Arena::CreateMessage>(&arena)); auto& field = *values.back(); for (int j = 0; j < kNumElems; ++j) { - field.Add(T{}); + field.Add(j); } } - size_t used_bytes_if_reusing = - values.size() * values[0]->Capacity() * (is_ptr ? sizeof(T*) : sizeof(T)); - // Use a 2% slack for other overhead. - // If we were not reusing the blocks, the actual value would be ~2x the - // expected. - EXPECT_THAT( - arena.SpaceUsed() - (is_ptr ? sizeof(T) * kNumElems * kNumFields : 0), - AllOf(Ge(used_bytes_if_reusing), Le(1.02 * used_bytes_if_reusing))); -} - -TEST(RepeatedField, NaturalGrowthOnArenasReuseBlocks) { - CheckNaturalGrowthOnArenasReuseBlocks>(false); + size_t expected = values.size() * values[0]->Capacity() * sizeof(int); + // Use a 2% slack for other overhead. If we were not reusing the blocks, the + // actual value would be ~2x the expected. + EXPECT_THAT(arena.SpaceUsed(), AllOf(Ge(expected), Le(1.02 * expected))); } // Test swapping between various types of RepeatedFields. @@ -1358,7 +1348,27 @@ TEST(RepeatedPtrField, ArenaAllocationSizesMatchExpectedValues) { } TEST(RepeatedPtrField, NaturalGrowthOnArenasReuseBlocks) { - CheckNaturalGrowthOnArenasReuseBlocks>(true); + using Rep = RepeatedPtrField; + Arena arena; + std::vector values; + + static constexpr int kNumFields = 100; + static constexpr int kNumElems = 1000; + for (int i = 0; i < kNumFields; ++i) { + values.push_back(Arena::CreateMessage(&arena)); + auto& field = *values.back(); + for (int j = 0; j < kNumElems; ++j) { + field.Add(""); + } + } + + size_t expected = + values.size() * values[0]->Capacity() * sizeof(std::string*) + + sizeof(std::string) * kNumElems * kNumFields; + // Use a 2% slack for other overhead. + // If we were not reusing the blocks, the actual value would be ~2x the + // expected. + EXPECT_THAT(arena.SpaceUsed(), AllOf(Ge(expected), Le(1.02 * expected))); } TEST(RepeatedPtrField, AddAndAssignRanges) { diff --git a/src/google/protobuf/serial_arena.h b/src/google/protobuf/serial_arena.h index 95767f664709..8ac5cddce10f 100644 --- a/src/google/protobuf/serial_arena.h +++ b/src/google/protobuf/serial_arena.h @@ -333,6 +333,19 @@ class PROTOBUF_EXPORT SerialArena { static size_t FreeStringBlocks(StringBlock* string_block, size_t unused); + // Adds 'used` to space_used_ in relaxed atomic order. + void AddSpaceUsed(size_t space_used) { + space_used_.store(space_used_.load(std::memory_order_relaxed) + space_used, + std::memory_order_relaxed); + } + + // Adds 'allocated` to space_allocated_ in relaxed atomic order. + void AddSpaceAllocated(size_t space_allocated) { + space_allocated_.store( + space_allocated_.load(std::memory_order_relaxed) + space_allocated, + std::memory_order_relaxed); + } + // Members are declared here to track sizeof(SerialArena) and hotness // centrally. They are (roughly) laid out in descending order of hotness. diff --git a/src/google/protobuf/string_block.h b/src/google/protobuf/string_block.h index fd7812b9fe8c..b105d57f3c11 100644 --- a/src/google/protobuf/string_block.h +++ b/src/google/protobuf/string_block.h @@ -62,11 +62,21 @@ class alignas(std::string) StringBlock { StringBlock(const StringBlock&) = delete; StringBlock& operator=(const StringBlock&) = delete; + // Returns the size of the next string block based on the size information + // stored in `block`. `block` may be null in which case the size of the + // initial string block is returned. + static size_t NextSize(StringBlock* block); + // Allocates a new StringBlock pointing to `next`, which can be null. // The size of the returned block depends on the allocated size of `next`. static StringBlock* New(StringBlock* next); - // Deletes `block`. `block` must not be null. + // Allocates a new string block `in place`. `n` must be the value returned + // from a previous call to `StringBlock::NextSize(next)` + static StringBlock* Emplace(void* p, size_t n, StringBlock* next); + + // Deletes `block` if `block` is heap allocated. `block` must not be null. + // Returns the allocated size of `block`, or 0 if the block was emplaced. static size_t Delete(StringBlock* block); StringBlock* next() const; @@ -86,6 +96,9 @@ class alignas(std::string) StringBlock { // Returns the total allocation size of this instance. size_t allocated_size() const { return allocated_size_; } + // Returns true if this block is heap allocated, false if emplaced. + bool heap_allocated() const { return heap_allocated_; } + // Returns the effective size available for allocation string instances. // This value is guaranteed to be a multiple of sizeof(std::string), and // guaranteed to never be zero. @@ -97,21 +110,44 @@ class alignas(std::string) StringBlock { ~StringBlock() = default; - explicit StringBlock(StringBlock* next, uint32_t size, + explicit StringBlock(StringBlock* next, bool heap_allocated, uint32_t size, uint32_t next_size) noexcept - : next_(next), allocated_size_(size), next_size_(next_size) {} + : next_(next), + heap_allocated_(heap_allocated), + allocated_size_(size), + next_size_(next_size) {} static constexpr uint32_t min_size() { return size_t{256}; } static constexpr uint32_t max_size() { return size_t{8192}; } + // Returns `size` rounded down such that we can fit a perfect number + // of std::string instances inside a StringBlock of that size. + static constexpr uint32_t RoundedSize(uint32_t size); + // Returns the size of the next block. size_t next_size() const { return next_size_; } StringBlock* const next_; - const uint32_t allocated_size_; + const bool heap_allocated_ : 1; + const uint32_t allocated_size_ : 31; const uint32_t next_size_; }; +constexpr uint32_t StringBlock::RoundedSize(uint32_t size) { + return size - (size - sizeof(StringBlock)) % sizeof(std::string); +} + +inline size_t StringBlock::NextSize(StringBlock* block) { + return block ? block->next_size() : min_size(); +} + +inline StringBlock* StringBlock::Emplace(void* p, size_t n, StringBlock* next) { + ABSL_DCHECK_EQ(n, NextSize(next)); + uint32_t doubled = static_cast(n) * 2; + uint32_t next_size = next ? std::min(doubled, max_size()) : min_size(); + return new (p) StringBlock(next, false, RoundedSize(n), next_size); +} + inline StringBlock* StringBlock::New(StringBlock* next) { // Compute required size, rounding down to a multiple of sizeof(std:string) // so that we can optimize the allocation path. I.e., we incur a (constant @@ -122,13 +158,14 @@ inline StringBlock* StringBlock::New(StringBlock* next) { size = next->next_size_; next_size = std::min(size * 2, max_size()); } - size -= (size - sizeof(StringBlock)) % sizeof(std::string); + size = RoundedSize(size); void* p = ::operator new(size); - return new (p) StringBlock(next, size, next_size); + return new (p) StringBlock(next, true, size, next_size); } inline size_t StringBlock::Delete(StringBlock* block) { ABSL_DCHECK(block != nullptr); + if (!block->heap_allocated_) return size_t{0}; size_t size = block->allocated_size(); internal::SizedDelete(block, size); return size; diff --git a/src/google/protobuf/string_block_test.cc b/src/google/protobuf/string_block_test.cc index ff1c71fd894d..3c4562d57d18 100644 --- a/src/google/protobuf/string_block_test.cc +++ b/src/google/protobuf/string_block_test.cc @@ -33,7 +33,9 @@ #include "google/protobuf/string_block.h" #include +#include #include +#include #include #include @@ -58,10 +60,12 @@ size_t AllocatedSizeFor(int size) { return EffectiveSizeFor(size) + sizeof(StringBlock); } -TEST(StringBlockTest, OneNewBlock) { +TEST(StringBlockTest, HeapAllocateOneBlock) { StringBlock* block = StringBlock::New(nullptr); + ASSERT_THAT(block, Ne(nullptr)); EXPECT_THAT(block->next(), Eq(nullptr)); + ASSERT_TRUE(block->heap_allocated()); EXPECT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(256))); EXPECT_THAT(block->effective_size(), Eq(EffectiveSizeFor(256))); EXPECT_THAT(block->begin(), Eq(block->AtOffset(0))); @@ -70,7 +74,26 @@ TEST(StringBlockTest, OneNewBlock) { EXPECT_THAT(StringBlock::Delete(block), Eq(AllocatedSizeFor(256))); } -TEST(StringBlockTest, NewBlocks) { +TEST(StringBlockTest, EmplaceOneBlock) { + // NextSize() returns unrounded 'min_size()' on first call. + size_t size = StringBlock::NextSize(nullptr); + EXPECT_THAT(size, Eq(256)); + + auto buffer = std::make_unique(size); + StringBlock* block = StringBlock::Emplace(buffer.get(), size, nullptr); + + ASSERT_THAT(block, Ne(nullptr)); + EXPECT_THAT(block->next(), Eq(nullptr)); + ASSERT_FALSE(block->heap_allocated()); + EXPECT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(256))); + EXPECT_THAT(block->effective_size(), Eq(EffectiveSizeFor(256))); + EXPECT_THAT(block->begin(), Eq(block->AtOffset(0))); + EXPECT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + + EXPECT_THAT(StringBlock::Delete(block), Eq(0)); +} + +TEST(StringBlockTest, HeapAllocateMultipleBlocks) { // Note: first two blocks are 256 StringBlock* previous = StringBlock::New(nullptr); @@ -78,6 +101,7 @@ TEST(StringBlockTest, NewBlocks) { StringBlock* block = StringBlock::New(previous); ASSERT_THAT(block, Ne(nullptr)); ASSERT_THAT(block->next(), Eq(previous)); + ASSERT_TRUE(block->heap_allocated()); ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(size))); ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(size))); ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); @@ -88,7 +112,8 @@ TEST(StringBlockTest, NewBlocks) { // Capped at 8K from here on StringBlock* block = StringBlock::New(previous); ASSERT_THAT(block, Ne(nullptr)); - EXPECT_THAT(block->next(), Eq(previous)); + ASSERT_THAT(block->next(), Eq(previous)); + ASSERT_TRUE(block->heap_allocated()); ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(8192))); ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(8192))); ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); @@ -102,6 +127,48 @@ TEST(StringBlockTest, NewBlocks) { } } +TEST(StringBlockTest, EmplaceMultipleBlocks) { + std::vector> buffers; + + // Convenience lambda to allocate a buffer and invoke Emplace on it. + auto EmplaceBlock = [&](StringBlock* previous) { + size_t size = StringBlock::NextSize(previous); + buffers.push_back(std::make_unique(size)); + return StringBlock::Emplace(buffers.back().get(), size, previous); + }; + + // Note: first two blocks are 256 + StringBlock* previous = EmplaceBlock(nullptr); + + for (int size = 256; size <= 8192; size *= 2) { + StringBlock* block = EmplaceBlock(previous); + ASSERT_THAT(block, Ne(nullptr)); + ASSERT_THAT(block->next(), Eq(previous)); + ASSERT_FALSE(block->heap_allocated()); + ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(size))); + ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(size))); + ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); + ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + previous = block; + } + + // Capped at 8K from here on + StringBlock* block = EmplaceBlock(previous); + ASSERT_THAT(block, Ne(nullptr)); + EXPECT_THAT(block->next(), Eq(previous)); + ASSERT_FALSE(block->heap_allocated()); + ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(8192))); + ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(8192))); + ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); + ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + + while (block) { + StringBlock* next = block->next(); + EXPECT_THAT(StringBlock::Delete(block), Eq(0)); + block = next; + } +} + } // namespace } // namespace internal } // namespace protobuf From df455a33fd61629fd87f8faa17e22ab42bd16ea1 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 09:52:57 -0800 Subject: [PATCH 064/463] Use correct wireformat type for repeated group fields. PiperOrigin-RevId: 510456181 --- src/google/protobuf/message_unittest.inc | 18 ++++++++++++++++++ src/google/protobuf/unittest.proto | 3 +++ 2 files changed, 21 insertions(+) diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 4305f5310c8c..24e67b520970 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -465,6 +465,24 @@ TEST(MESSAGE_TEST_NAME, ParseFailsIfGroupFieldMalformed) { EXPECT_FALSE(parsed.ParseFromString(data)); } +TEST(MESSAGE_TEST_NAME, ParseFailsIfRepeatedGroupFieldMalformed) { + UNITTEST::TestMutualRecursionA original, parsed; + original.mutable_bb() + ->mutable_a() + ->add_subgroupr() + ->mutable_payload() + ->set_optional_int64(-1); + + std::string data; + ASSERT_TRUE(original.SerializeToString(&data)); + // Should parse correctly. + ASSERT_TRUE(parsed.ParseFromString(data)); + // Overwriting the last byte of varint (-1) to 0xFF results in malformed wire. + data[data.size() - 2] = 0xFF; + + EXPECT_FALSE(parsed.ParseFromString(data)); +} + TEST(MESSAGE_TEST_NAME, UninitializedAndMalformed) { UNITTEST::TestRequiredForeign o, p1, p2; o.mutable_optional_message()->set_a(-1); diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 479262f61272..7ebd53f9a2ce 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -552,6 +552,9 @@ message TestMutualRecursionA { optional SubMessage sub_message = 3; // Needed because of bug in javatest optional TestAllTypes not_in_this_scc = 4; } + repeated group SubGroupR = 5 { + optional TestAllTypes payload = 6; + } } message TestMutualRecursionB { From b4370b2e762b9f384b61a0a0a6e607349e92aa1a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 17 Feb 2023 12:10:22 -0800 Subject: [PATCH 065/463] [ObjC] Add -[GPBMessage mergeFromData:extensionRegistry:error:] This adds a merge method that doesn't throw, making it generally more usable and allowing usage directly from Swift. PiperOrigin-RevId: 510490798 --- objectivec/GPBMessage.h | 16 ++++++++++++++++ objectivec/GPBMessage.m | 36 ++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/objectivec/GPBMessage.h b/objectivec/GPBMessage.h index d9fc8986a2b7..690bf0390aeb 100644 --- a/objectivec/GPBMessage.h +++ b/objectivec/GPBMessage.h @@ -278,6 +278,22 @@ CF_EXTERN_C_END - (void)mergeFromData:(NSData *)data extensionRegistry:(nullable id)extensionRegistry; +/** + * Parses the given data as this message's class, and merges those values into + * this message. + * + * @param data The binary representation of the message to merge. + * @param extensionRegistry The extension registry to use to look up extensions. + * @param errorPtr An optional error pointer to fill in with a failure + * reason if the data can not be parsed. Will only be + * filled in if the data failed to be parsed. + * + * @return Boolean indicating success. errorPtr will only be fill in on failure. + **/ +- (BOOL)mergeFromData:(NSData *)data + extensionRegistry:(nullable id)extensionRegistry + error:(NSError **)errorPtr; + /** * Merges the fields from another message (of the same type) into this * message. diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 0bdf4bb8a777..0793c9030d6c 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -965,27 +965,18 @@ - (instancetype)initWithData:(NSData *)data extensionRegistry:(id)extensionRegistry error:(NSError **)errorPtr { if ((self = [self init])) { - @try { - [self mergeFromData:data extensionRegistry:extensionRegistry]; - if (errorPtr) { - *errorPtr = nil; - } - } @catch (NSException *exception) { + if (![self mergeFromData:data extensionRegistry:extensionRegistry error:errorPtr]) { [self release]; self = nil; - if (errorPtr) { - *errorPtr = ErrorFromException(exception); - } - } #ifdef DEBUG - if (self && !self.initialized) { + } else if (!self.initialized) { [self release]; self = nil; if (errorPtr) { *errorPtr = MessageError(GPBMessageErrorCodeMissingRequiredField, nil); } - } #endif + } } return self; } @@ -2004,6 +1995,27 @@ - (void)mergeFromData:(NSData *)data extensionRegistry:(id } } +- (BOOL)mergeFromData:(NSData *)data + extensionRegistry:(nullable id)extensionRegistry + error:(NSError **)errorPtr { + GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; + @try { + [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; + [input checkLastTagWas:0]; + if (errorPtr) { + *errorPtr = nil; + } + } @catch (NSException *exception) { + [input release]; + if (errorPtr) { + *errorPtr = ErrorFromException(exception); + } + return NO; + } + [input release]; + return YES; +} + #pragma mark - Parse From Data Support + (instancetype)parseFromData:(NSData *)data error:(NSError **)errorPtr { From 47b5f81c7f944b23a59b6ee9b0f0d2e9a517e10b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 10:56:21 -0800 Subject: [PATCH 066/463] Internal change. PiperOrigin-RevId: 511247805 --- src/google/protobuf/port_def.inc | 4 ++++ src/google/protobuf/repeated_ptr_field.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 3283030c16dd..5935e7f45a65 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -213,6 +213,10 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // Owner: shaod@, gberg@ #define PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL 1 +// Used to remove `RepeatedPtrField::GetArena() const`. +// Owner: ezb@ +#define PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API 1 + #endif #ifdef PROTOBUF_VERSION diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index bb5ed22ea511..2265cf5dccb5 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -1185,8 +1185,10 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { // Gets the arena on which this RepeatedPtrField stores its elements. inline Arena* GetArena(); +#ifndef PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API ABSL_DEPRECATED("This will be removed in a future release") inline Arena* GetArena() const; +#endif // !PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API // For internal use only. // @@ -1552,10 +1554,12 @@ inline Arena* RepeatedPtrField::GetArena() { return RepeatedPtrFieldBase::GetArena(); } +#ifndef PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API template inline Arena* RepeatedPtrField::GetArena() const { return RepeatedPtrFieldBase::GetArena(); } +#endif // !PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API template inline Arena* RepeatedPtrField::GetOwningArena() const { From 9b5a2c6f6fc76da3c93f6793550fa488ae62abaf Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Tue, 21 Feb 2023 11:11:05 -0800 Subject: [PATCH 067/463] Fix pom_template dependencies. All pom files need a dependency section and kotlin + util need to depend on protobuf-java and kotlin-lite should depend on protobuf-javalite. This fixes https://github.com/protocolbuffers/protobuf/issues/11976 PiperOrigin-RevId: 511252224 --- java/core/pom_template.xml | 3 +++ java/kotlin-lite/pom_template.xml | 7 +++++++ java/kotlin/pom_template.xml | 7 +++++++ java/lite/pom_template.xml | 3 +++ java/util/pom_template.xml | 4 ++++ 5 files changed, 24 insertions(+) diff --git a/java/core/pom_template.xml b/java/core/pom_template.xml index 2c61281b0d14..4a30ddc86343 100644 --- a/java/core/pom_template.xml +++ b/java/core/pom_template.xml @@ -15,4 +15,7 @@ Core Protocol Buffers library. Protocol Buffers are a way of encoding structured data in an efficient yet extensible format. + + {dependencies} + diff --git a/java/kotlin-lite/pom_template.xml b/java/kotlin-lite/pom_template.xml index f8a339740831..90d9857dd897 100644 --- a/java/kotlin-lite/pom_template.xml +++ b/java/kotlin-lite/pom_template.xml @@ -15,6 +15,13 @@ Kotlin lite Protocol Buffers library. Protocol Buffers are a way of encoding structured data in an efficient yet extensible format. + + + ${groupId} + protobuf-javalite + + {dependencies} + 1.6.0 diff --git a/java/kotlin/pom_template.xml b/java/kotlin/pom_template.xml index 0feae33e54f9..d04bc89fc371 100644 --- a/java/kotlin/pom_template.xml +++ b/java/kotlin/pom_template.xml @@ -15,6 +15,13 @@ Kotlin core Protocol Buffers library. Protocol Buffers are a way of encoding structured data in an efficient yet extensible format. + + + ${groupId} + protobuf-java + + {dependencies} + 1.6.0 diff --git a/java/lite/pom_template.xml b/java/lite/pom_template.xml index 6d705fdcbe08..fa6cfd2ae6b9 100644 --- a/java/lite/pom_template.xml +++ b/java/lite/pom_template.xml @@ -15,5 +15,8 @@ Lite version of Protocol Buffers library. This version is optimized for code size, but does not guarantee API/ABI stability. + + {dependencies} + diff --git a/java/util/pom_template.xml b/java/util/pom_template.xml index 13fe3ecf1c7c..f897892510b0 100644 --- a/java/util/pom_template.xml +++ b/java/util/pom_template.xml @@ -13,6 +13,10 @@ Protocol Buffers [Util] Utilities for Protocol Buffers + + ${groupId} + protobuf-java + {dependencies} From 68b8b43d42612336d093645d8cc0e9cc581e7ca2 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Tue, 21 Feb 2023 11:21:18 -0800 Subject: [PATCH 068/463] Allow empty commits for update_php_repo Fix exit code on git commit which was preventing force-push and tagging from running even though these should work without a new commit. An empty commit is probably clearer anyways to show that sync has happened. PiperOrigin-RevId: 511255365 --- .github/workflows/update_php_repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_php_repo.yml b/.github/workflows/update_php_repo.yml index 548cb6eefd40..cb3383ad019d 100644 --- a/.github/workflows/update_php_repo.yml +++ b/.github/workflows/update_php_repo.yml @@ -43,7 +43,7 @@ jobs: - name: Push Changes run: | git add --all - git commit -m "${{ env.VERSION }} sync" + git commit --allow-empty -m "${{ env.VERSION }} sync" git push --force origin master git tag -a ${{ env.VERSION_TAG }} -m "Tag release ${{ env.VERSION_TAG }}" git push origin ${{ env.VERSION_TAG }} From cffde99fc6195ca108f320c16a6ded41c42a3317 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 11:40:52 -0800 Subject: [PATCH 069/463] [ObjC] Enforcement of the 2GB limit when parsing messages. Add a helper to put the logic in one place in the file. Ensure all places that read messages also do the 2GB check for message size. PiperOrigin-RevId: 511260874 --- objectivec/GPBCodedInputStream.m | 49 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index 9acf6f170fca..c8464a29209f 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -51,10 +51,6 @@ // int CodedInputStream::default_recursion_limit_ = 100; static const NSUInteger kDefaultRecursionLimit = 100; -// Bytes and Strings have a max size of 2GB. -// https://protobuf.dev/programming-guides/encoding/#cheat-sheet -static const uint32_t kMaxFieldSize = 0x7fffffff; - static void RaiseException(NSInteger code, NSString *reason) { NSDictionary *errorInfo = nil; if ([reason length]) { @@ -69,12 +65,24 @@ static void RaiseException(NSInteger code, NSString *reason) { userInfo:exceptionInfo] raise]; } -static void CheckRecursionLimit(GPBCodedInputStreamState *state) { +GPB_INLINE void CheckRecursionLimit(GPBCodedInputStreamState *state) { if (state->recursionDepth >= kDefaultRecursionLimit) { RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } } +GPB_INLINE void CheckFieldSize(uint64_t size) { + // Bytes and Strings have a max size of 2GB. And since messages are on the wire as bytes/length + // delimited, they also have a 2GB size limit. The C++ does the same sort of enforcement (see + // parse_context, delimited_message_util, message_lite, etc.). + // https://protobuf.dev/programming-guides/encoding/#cheat-sheet + if (size > 0x7fffffff) { + // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking + // change so reuse an existing one. + RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + } +} + static void CheckSize(GPBCodedInputStreamState *state, size_t size) { size_t newSize = state->bufferPos + size; if (newSize > state->bufferSize) { @@ -228,11 +236,7 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { NSString *GPBCodedInputStreamReadRetainedString(GPBCodedInputStreamState *state) { uint64_t size = GPBCodedInputStreamReadUInt64(state); - if (size > kMaxFieldSize) { - // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change - // so reuse an existing one. - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); - } + CheckFieldSize(size); NSUInteger ns_size = (NSUInteger)size; NSString *result; if (size == 0) { @@ -257,11 +261,7 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state) { uint64_t size = GPBCodedInputStreamReadUInt64(state); - if (size > kMaxFieldSize) { - // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change - // so reuse an existing one. - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); - } + CheckFieldSize(size); NSUInteger ns_size = (NSUInteger)size; CheckSize(state, size); NSData *result = [[NSData alloc] initWithBytes:state->bytes + state->bufferPos length:ns_size]; @@ -271,11 +271,7 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state) { uint64_t size = GPBCodedInputStreamReadUInt64(state); - if (size > kMaxFieldSize) { - // TODO(thomasvl): Maybe a different error code for this, but adding one is a breaking change - // so reuse an existing one. - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); - } + CheckFieldSize(size); NSUInteger ns_size = (NSUInteger)size; CheckSize(state, size); // Cast is safe because freeWhenDone is NO. @@ -362,9 +358,12 @@ - (BOOL)skipField:(int32_t)tag { case GPBWireFormatFixed64: SkipRawData(&state_, sizeof(int64_t)); return YES; - case GPBWireFormatLengthDelimited: - SkipRawData(&state_, ReadRawVarint32(&state_)); + case GPBWireFormatLengthDelimited: { + uint64_t size = GPBCodedInputStreamReadUInt64(&state_); + CheckFieldSize(size); + SkipRawData(&state_, size); return YES; + } case GPBWireFormatStartGroup: [self skipMessage]; GPBCodedInputStreamCheckLastTagWas( @@ -463,7 +462,8 @@ - (void)readUnknownGroup:(int32_t)fieldNumber message:(GPBUnknownFieldSet *)mess - (void)readMessage:(GPBMessage *)message extensionRegistry:(id)extensionRegistry { CheckRecursionLimit(&state_); - int32_t length = ReadRawVarint32(&state_); + uint64_t length = GPBCodedInputStreamReadUInt64(&state_); + CheckFieldSize(length); size_t oldLimit = GPBCodedInputStreamPushLimit(&state_, length); ++state_.recursionDepth; [message mergeFromCodedInputStream:self extensionRegistry:extensionRegistry]; @@ -477,7 +477,8 @@ - (void)readMapEntry:(id)mapDictionary field:(GPBFieldDescriptor *)field parentMessage:(GPBMessage *)parentMessage { CheckRecursionLimit(&state_); - int32_t length = ReadRawVarint32(&state_); + uint64_t length = GPBCodedInputStreamReadUInt64(&state_); + CheckFieldSize(length); size_t oldLimit = GPBCodedInputStreamPushLimit(&state_, length); ++state_.recursionDepth; GPBDictionaryReadEntry(mapDictionary, self, extensionRegistry, field, parentMessage); From e3b00511099838e22f59827bfb7c72e27fcc22fa Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 11:45:16 -0800 Subject: [PATCH 070/463] [ObjC] mark mergeFromData:extensionRegistry: as deprecated. Since it really isn't safe in Swift, point folks at the new api that uses errors. PiperOrigin-RevId: 511262228 --- objectivec/GPBMessage.h | 4 +++- objectivec/GPBMessage.m | 3 +++ objectivec/Tests/GPBMessageTests.m | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/objectivec/GPBMessage.h b/objectivec/GPBMessage.h index 690bf0390aeb..ae2dd71eecbb 100644 --- a/objectivec/GPBMessage.h +++ b/objectivec/GPBMessage.h @@ -276,7 +276,9 @@ CF_EXTERN_C_END * unsuccessful. **/ - (void)mergeFromData:(NSData *)data - extensionRegistry:(nullable id)extensionRegistry; + extensionRegistry:(nullable id)extensionRegistry + __attribute__((deprecated( + "Use -mergeFromData:extensionRegistry:error: instead, especaily if calling from Swift."))); /** * Parses the given data as this message's class, and merges those values into diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 0793c9030d6c..c287af7c8a34 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -3261,7 +3261,10 @@ - (instancetype)initWithCoder:(NSCoder *)aDecoder { if (self) { NSData *data = [aDecoder decodeObjectOfClass:[NSData class] forKey:kGPBDataCoderKey]; if (data.length) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" [self mergeFromData:data extensionRegistry:nil]; +#pragma clang diagnostic pop } } return self; diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m index 6a1d48beda4b..2e595f6aa1ae 100644 --- a/objectivec/Tests/GPBMessageTests.m +++ b/objectivec/Tests/GPBMessageTests.m @@ -224,7 +224,11 @@ - (void)testMergeFromWithExtensions { result = [self mergeExtensionsDestination]; NSData *data = [[self mergeExtensionsSource] data]; XCTAssertNotNil(data); - [result mergeFromData:data extensionRegistry:[UnittestRoot extensionRegistry]]; + NSError *error = nil; + XCTAssertTrue([result mergeFromData:data + extensionRegistry:[UnittestRoot extensionRegistry] + error:&error]); + XCTAssertNil(error); resultData = [result data]; XCTAssertEqualObjects(resultData, mergeResultData); XCTAssertEqualObjects(result, [self mergeExtensionsResult]); From b51858aa1180a62257dae069a3a6ee21cd1b411c Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Tue, 21 Feb 2023 15:07:12 -0800 Subject: [PATCH 071/463] Document that hasField is only semantically meaningful when hasPresence == True. Note that this is already documented in hasPresence as well. PiperOrigin-RevId: 511316345 --- .../src/main/java/com/google/protobuf/MessageOrBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/google/protobuf/MessageOrBuilder.java b/java/core/src/main/java/com/google/protobuf/MessageOrBuilder.java index 2a4d867ae164..267507008239 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageOrBuilder.java +++ b/java/core/src/main/java/com/google/protobuf/MessageOrBuilder.java @@ -95,7 +95,8 @@ public interface MessageOrBuilder extends MessageLiteOrBuilder { /** * Returns true if the given field is set. This is exactly equivalent to calling the generated - * "has" accessor method corresponding to the field. + * "has" accessor method corresponding to the field. The return value of hasField() is + * semantically meaningful only for fields where field.hasPresence() == true. * * @throws IllegalArgumentException The field is a repeated field, or {@code * field.getContainingType() != getDescriptorForType()}. From 7958aa90ce7aeee2c578c763b40908477cd4cc79 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 21 Feb 2023 15:18:48 -0800 Subject: [PATCH 072/463] Check for invalid escaping when parsing the default value of a bytes field. The code that loads descriptors calls absl::CUnescape() to parse the escape sequences. Prior to this CL, we were not checking the return value of this call, effectively allowing the invalid data to be loaded. If a user is compiling a .proto file, invalid escapes will be caught earlier in the compiler pipeline, and this check will be unnecessary. The only time this is an issue is when a descriptor has been constructed manually and is loaded into the runtime. This bug manifested as an MSAN failure in a fuzz test, because in the error case absl::CUnescape() will return a string where some of the characters are uninitialized. PiperOrigin-RevId: 511319279 --- src/google/protobuf/descriptor.cc | 9 +++++++-- src/google/protobuf/descriptor_unittest.cc | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index efcd86e35895..ecffebe56996 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -5782,8 +5782,13 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, case FieldDescriptor::CPPTYPE_STRING: if (result->type() == FieldDescriptor::TYPE_BYTES) { std::string value; - absl::CUnescape(proto.default_value(), &value); - result->default_value_string_ = alloc.AllocateStrings(value); + if (absl::CUnescape(proto.default_value(), &value)) { + result->default_value_string_ = alloc.AllocateStrings(value); + } else { + AddError(result->full_name(), proto, + DescriptorPool::ErrorCollector::DEFAULT_VALUE, + "Invalid escaping in default value."); + } } else { result->default_value_string_ = alloc.AllocateStrings(proto.default_value()); diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 64e6b2c52ad7..fce00a6566e3 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -4558,6 +4558,12 @@ TEST_F(ValidationErrorTest, InvalidDefaults) { " field { name: \"corge\" number: 6 label: LABEL_REPEATED type: " "TYPE_INT32" " default_value: \"1\" }" + + // Invalid CEscaped bytes default. + " field { name: \"bytes_default\" number: 7 label: LABEL_OPTIONAL " + " type: TYPE_BYTES" + " default_value: \"\\\\\" }" + "}", "foo.proto: Foo.foo: DEFAULT_VALUE: Couldn't parse default value " @@ -4568,6 +4574,8 @@ TEST_F(ValidationErrorTest, InvalidDefaults) { "foo.proto: Foo.moo: DEFAULT_VALUE: Messages can't have default values.\n" "foo.proto: Foo.corge: DEFAULT_VALUE: Repeated fields can't have default " "values.\n" + "foo.proto: Foo.bytes_default: DEFAULT_VALUE: Invalid escaping in " + "default value.\n" // This ends up being reported later because the error is detected at // cross-linking time. "foo.proto: Foo.mooo: DEFAULT_VALUE: Messages can't have default " From 5995ad26a564379b44ab2ba08c89de62cf7c5942 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 16:51:49 -0800 Subject: [PATCH 073/463] Internal change. PiperOrigin-RevId: 511341281 --- src/google/protobuf/extension_set.h | 2 ++ src/google/protobuf/wire_format.cc | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index d5ef61a7e803..712b5960ae4c 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -75,6 +75,7 @@ class Reflection; // message.h class UnknownFieldSet; // unknown_field_set.h namespace internal { class FieldSkipper; // wire_format_lite.h +class WireFormat; enum class LazyVerifyOption; } // namespace internal } // namespace protobuf @@ -514,6 +515,7 @@ class PROTOBUF_EXPORT ExtensionSet { friend class RepeatedEnumTypeTraits; friend class google::protobuf::Reflection; + friend class google::protobuf::internal::WireFormat; const int32_t& GetRefInt32(int number, const int32_t& default_value) const; const int64_t& GetRefInt64(int number, const int64_t& default_value) const; diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index 27aa64fda93a..e0c18bb00cf6 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -1466,11 +1466,11 @@ uint8_t* WireFormat::InternalSerializeMessageSetItem( // Write type ID. target = WireFormatLite::WriteUInt32ToArray( WireFormatLite::kMessageSetTypeIdNumber, field->number(), target); - // Write message. - auto& msg = message_reflection->GetMessage(message, field); - target = WireFormatLite::InternalWriteMessage( - WireFormatLite::kMessageSetMessageNumber, msg, msg.GetCachedSize(), - target, stream); + // Write message. + auto& msg = message_reflection->GetMessage(message, field); + target = WireFormatLite::InternalWriteMessage( + WireFormatLite::kMessageSetMessageNumber, msg, msg.GetCachedSize(), + target, stream); // End group. target = stream->EnsureSpace(target); target = io::CodedOutputStream::WriteTagToArray( From 57d5ede3ec807851ed9293f96aff8651b64f1e4d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 17:44:36 -0800 Subject: [PATCH 074/463] Rollback PiperOrigin-RevId: 511351345 --- src/google/protobuf/descriptor.cc | 9 ++------- src/google/protobuf/descriptor_unittest.cc | 8 -------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index ecffebe56996..efcd86e35895 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -5782,13 +5782,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, case FieldDescriptor::CPPTYPE_STRING: if (result->type() == FieldDescriptor::TYPE_BYTES) { std::string value; - if (absl::CUnescape(proto.default_value(), &value)) { - result->default_value_string_ = alloc.AllocateStrings(value); - } else { - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::DEFAULT_VALUE, - "Invalid escaping in default value."); - } + absl::CUnescape(proto.default_value(), &value); + result->default_value_string_ = alloc.AllocateStrings(value); } else { result->default_value_string_ = alloc.AllocateStrings(proto.default_value()); diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index fce00a6566e3..64e6b2c52ad7 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -4558,12 +4558,6 @@ TEST_F(ValidationErrorTest, InvalidDefaults) { " field { name: \"corge\" number: 6 label: LABEL_REPEATED type: " "TYPE_INT32" " default_value: \"1\" }" - - // Invalid CEscaped bytes default. - " field { name: \"bytes_default\" number: 7 label: LABEL_OPTIONAL " - " type: TYPE_BYTES" - " default_value: \"\\\\\" }" - "}", "foo.proto: Foo.foo: DEFAULT_VALUE: Couldn't parse default value " @@ -4574,8 +4568,6 @@ TEST_F(ValidationErrorTest, InvalidDefaults) { "foo.proto: Foo.moo: DEFAULT_VALUE: Messages can't have default values.\n" "foo.proto: Foo.corge: DEFAULT_VALUE: Repeated fields can't have default " "values.\n" - "foo.proto: Foo.bytes_default: DEFAULT_VALUE: Invalid escaping in " - "default value.\n" // This ends up being reported later because the error is detected at // cross-linking time. "foo.proto: Foo.mooo: DEFAULT_VALUE: Messages can't have default " From c8e005db3a73c26d6597dcc42b3060e277a442b6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 21 Feb 2023 21:14:34 -0800 Subject: [PATCH 075/463] Internal change. PiperOrigin-RevId: 511387968 --- src/google/protobuf/extension_set.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 712b5960ae4c..99b0e397f8fc 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -562,6 +562,8 @@ class PROTOBUF_EXPORT ExtensionSet { virtual bool IsInitialized(const MessageLite* prototype, Arena* arena) const = 0; + virtual bool IsEagerSerializeSafe(const MessageLite* prototype, + Arena* arena) const = 0; PROTOBUF_DEPRECATED_MSG("Please use ByteSizeLong() instead") virtual int ByteSize() const { return internal::ToIntSize(ByteSizeLong()); } From e6d01b2edcb04fdfb0f3cf79bf9d427f57fa2eac Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 22 Feb 2023 05:54:19 -0800 Subject: [PATCH 076/463] [ObjC] Enforce the max message size when serializing to binary form. The validation is done at the highest point so if a sub message is what goes over the limit it is caught at the outer message, thus reducing the impact on the serialization code. PiperOrigin-RevId: 511473008 --- objectivec/GPBCodedOutputStream.h | 5 ++ objectivec/GPBCodedOutputStream.m | 10 +++ objectivec/GPBMessage.h | 6 ++ objectivec/GPBMessage.m | 64 ++++++++++++++++---- objectivec/Tests/GPBCodedOutputStreamTests.m | 20 ++++++ 5 files changed, 93 insertions(+), 12 deletions(-) diff --git a/objectivec/GPBCodedOutputStream.h b/objectivec/GPBCodedOutputStream.h index 42eac77044b8..4969e5a8af7c 100644 --- a/objectivec/GPBCodedOutputStream.h +++ b/objectivec/GPBCodedOutputStream.h @@ -109,6 +109,11 @@ __attribute__((objc_subclassing_restricted)) **/ - (void)flush; +/** + * @return The number of bytes written out. Includes bytes not yet flused. + **/ +- (size_t)bytesWritten; + /** * Write the raw byte out. * diff --git a/objectivec/GPBCodedOutputStream.m b/objectivec/GPBCodedOutputStream.m index c21e7d26f679..da6271535d1f 100644 --- a/objectivec/GPBCodedOutputStream.m +++ b/objectivec/GPBCodedOutputStream.m @@ -48,6 +48,7 @@ uint8_t *bytes; size_t size; size_t position; + size_t bytesFlushed; NSOutputStream *output; } GPBOutputBufferState; @@ -71,6 +72,7 @@ static void GPBRefreshBuffer(GPBOutputBufferState *state) { if (written != (NSInteger)state->position) { [NSException raise:GPBCodedOutputStreamException_WriteFailed format:@""]; } + state->bytesFlushed += written; state->position = 0; } } @@ -192,6 +194,13 @@ + (instancetype)streamWithData:(NSMutableData *)data { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdirect-ivar-access" +- (size_t)bytesWritten { + // Could use NSStreamFileCurrentOffsetKey on state_.output if there is a stream, that could be + // expensive, manually tracking what is flush keeps things faster so message serialization can + // check it. + return state_.bytesFlushed + state_.position; +} + - (void)writeDoubleNoTag:(double)value { GPBWriteRawLittleEndian64(&state_, GPBConvertDoubleToInt64(value)); } @@ -886,6 +895,7 @@ - (void)writeRawPtr:(const void *)value offset:(size_t)offset length:(size_t)len if (written != (NSInteger)length) { [NSException raise:GPBCodedOutputStreamException_WriteFailed format:@""]; } + state_.bytesFlushed += written; } } } diff --git a/objectivec/GPBMessage.h b/objectivec/GPBMessage.h index ae2dd71eecbb..5d55df1c4051 100644 --- a/objectivec/GPBMessage.h +++ b/objectivec/GPBMessage.h @@ -61,6 +61,12 @@ typedef NS_ENUM(NSInteger, GPBMessageErrorCode) { **/ extern NSString *const GPBErrorReasonKey; +/** + * An exception name raised during serialization when the message would be + * larger than the 2GB limit. + **/ +extern NSString *const GPBMessageExceptionMessageTooLarge; + CF_EXTERN_C_END /** diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index c287af7c8a34..a284666808d7 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -59,6 +59,16 @@ static NSString *const kGPBDataCoderKey = @"GPBData"; +// Length-delimited has a max size of 2GB, and thus messages do also. +// src/google/protobuf/message_lite also does this enforcement on the C++ side. Validation for +// parsing is done with GPBCodedInputStream; but for messages, it is less checks to do it within +// the message side since the input stream code calls these same bottlenecks. +// https://protobuf.dev/programming-guides/encoding/#cheat-sheet +static const size_t kMaximumMessageSize = 0x7fffffff; + +NSString *const GPBMessageExceptionMessageTooLarge = + GPBNSStringifySymbol(GPBMessageExceptionMessageTooLarge); + // // PLEASE REMEMBER: // @@ -111,7 +121,7 @@ static id CreateMapForField(GPBFieldDescriptor *field, GPBMessage *autocreator) __attribute__((ns_returns_retained)); static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self); -#ifdef DEBUG +#if defined(DEBUG) && DEBUG static NSError *MessageError(NSInteger code, NSDictionary *userInfo) { return [NSError errorWithDomain:GPBMessageErrorDomain code:code userInfo:userInfo]; } @@ -968,7 +978,7 @@ - (instancetype)initWithData:(NSData *)data if (![self mergeFromData:data extensionRegistry:extensionRegistry error:errorPtr]) { [self release]; self = nil; -#ifdef DEBUG +#if defined(DEBUG) && DEBUG } else if (!self.initialized) { [self release]; self = nil; @@ -997,7 +1007,7 @@ - (instancetype)initWithCodedInputStream:(GPBCodedInputStream *)input *errorPtr = ErrorFromException(exception); } } -#ifdef DEBUG +#if defined(DEBUG) && DEBUG if (self && !self.initialized) { [self release]; self = nil; @@ -1290,12 +1300,16 @@ - (GPBDescriptor *)descriptor { } - (NSData *)data { -#ifdef DEBUG +#if defined(DEBUG) && DEBUG if (!self.initialized) { return nil; } #endif - NSMutableData *data = [NSMutableData dataWithLength:[self serializedSize]]; + size_t expectedSize = [self serializedSize]; + if (expectedSize > kMaximumMessageSize) { + return nil; + } + NSMutableData *data = [NSMutableData dataWithLength:expectedSize]; GPBCodedOutputStream *stream = [[GPBCodedOutputStream alloc] initWithData:data]; @try { [self writeToCodedOutputStream:stream]; @@ -1306,11 +1320,14 @@ - (NSData *)data { // to this message or a message used as a nested field, and that other thread mutated that // message, causing the pre computed serializedSize to no longer match the final size after // serialization. It is not safe to mutate a message while accessing it from another thread. -#ifdef DEBUG +#if defined(DEBUG) && DEBUG NSLog(@"%@: Internal exception while building message data: %@", [self class], exception); #endif data = nil; } +#if defined(DEBUG) && DEBUG + NSAssert(!data || [stream bytesWritten] == expectedSize, @"Internal error within the library"); +#endif [stream release]; return data; } @@ -1329,7 +1346,7 @@ - (NSData *)delimitedData { // to this message or a message used as a nested field, and that other thread mutated that // message, causing the pre computed serializedSize to no longer match the final size after // serialization. It is not safe to mutate a message while accessing it from another thread. -#ifdef DEBUG +#if defined(DEBUG) && DEBUG NSLog(@"%@: Internal exception while building message delimitedData: %@", [self class], exception); #endif @@ -1342,8 +1359,16 @@ - (NSData *)delimitedData { - (void)writeToOutputStream:(NSOutputStream *)output { GPBCodedOutputStream *stream = [[GPBCodedOutputStream alloc] initWithOutputStream:output]; - [self writeToCodedOutputStream:stream]; - [stream release]; + @try { + [self writeToCodedOutputStream:stream]; + size_t bytesWritten = [stream bytesWritten]; + if (bytesWritten > kMaximumMessageSize) { + [NSException raise:GPBMessageExceptionMessageTooLarge + format:@"Message would have been %zu bytes", bytesWritten]; + } + } @finally { + [stream release]; + } } - (void)writeToCodedOutputStream:(GPBCodedOutputStream *)output { @@ -1377,13 +1402,28 @@ - (void)writeToCodedOutputStream:(GPBCodedOutputStream *)output { - (void)writeDelimitedToOutputStream:(NSOutputStream *)output { GPBCodedOutputStream *codedOutput = [[GPBCodedOutputStream alloc] initWithOutputStream:output]; - [self writeDelimitedToCodedOutputStream:codedOutput]; - [codedOutput release]; + @try { + [self writeDelimitedToCodedOutputStream:codedOutput]; + } @finally { + [codedOutput release]; + } } - (void)writeDelimitedToCodedOutputStream:(GPBCodedOutputStream *)output { - [output writeRawVarintSizeTAs32:[self serializedSize]]; + size_t expectedSize = [self serializedSize]; + if (expectedSize > kMaximumMessageSize) { + [NSException raise:GPBMessageExceptionMessageTooLarge + format:@"Message would have been %zu bytes", expectedSize]; + } + [output writeRawVarintSizeTAs32:expectedSize]; +#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) + size_t initialSize = [output bytesWritten]; +#endif [self writeToCodedOutputStream:output]; +#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) + NSAssert(([output bytesWritten] - initialSize) == expectedSize, + @"Internal error within the library"); +#endif } - (void)writeField:(GPBFieldDescriptor *)field toCodedOutputStream:(GPBCodedOutputStream *)output { diff --git a/objectivec/Tests/GPBCodedOutputStreamTests.m b/objectivec/Tests/GPBCodedOutputStreamTests.m index a619cae05040..6218f518d64e 100644 --- a/objectivec/Tests/GPBCodedOutputStreamTests.m +++ b/objectivec/Tests/GPBCodedOutputStreamTests.m @@ -80,7 +80,9 @@ - (void)assertWriteLittleEndian32:(NSData*)data value:(int32_t)value { NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory]; GPBCodedOutputStream* output = [GPBCodedOutputStream streamWithOutputStream:rawOutput]; [output writeRawLittleEndian32:(int32_t)value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -90,7 +92,9 @@ - (void)assertWriteLittleEndian32:(NSData*)data value:(int32_t)value { rawOutput = [NSOutputStream outputStreamToMemory]; output = [GPBCodedOutputStream streamWithOutputStream:rawOutput bufferSize:blockSize]; [output writeRawLittleEndian32:(int32_t)value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -101,7 +105,9 @@ - (void)assertWriteLittleEndian64:(NSData*)data value:(int64_t)value { NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory]; GPBCodedOutputStream* output = [GPBCodedOutputStream streamWithOutputStream:rawOutput]; [output writeRawLittleEndian64:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -111,7 +117,9 @@ - (void)assertWriteLittleEndian64:(NSData*)data value:(int64_t)value { rawOutput = [NSOutputStream outputStreamToMemory]; output = [GPBCodedOutputStream streamWithOutputStream:rawOutput bufferSize:blockSize]; [output writeRawLittleEndian64:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -124,7 +132,9 @@ - (void)assertWriteVarint:(NSData*)data value:(int64_t)value { NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory]; GPBCodedOutputStream* output = [GPBCodedOutputStream streamWithOutputStream:rawOutput]; [output writeRawVarint32:(int32_t)value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -137,7 +147,9 @@ - (void)assertWriteVarint:(NSData*)data value:(int64_t)value { NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory]; GPBCodedOutputStream* output = [GPBCodedOutputStream streamWithOutputStream:rawOutput]; [output writeRawVarint64:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -155,7 +167,9 @@ - (void)assertWriteVarint:(NSData*)data value:(int64_t)value { bufferSize:blockSize]; [output writeRawVarint32:(int32_t)value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -167,7 +181,9 @@ - (void)assertWriteVarint:(NSData*)data value:(int64_t)value { bufferSize:blockSize]; [output writeRawVarint64:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual); @@ -181,7 +197,9 @@ - (void)assertWriteStringNoTag:(NSData*)data NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory]; GPBCodedOutputStream* output = [GPBCodedOutputStream streamWithOutputStream:rawOutput]; [output writeStringNoTag:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); NSData* actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual, @"%@", contextMessage); @@ -191,7 +209,9 @@ - (void)assertWriteStringNoTag:(NSData*)data rawOutput = [NSOutputStream outputStreamToMemory]; output = [GPBCodedOutputStream streamWithOutputStream:rawOutput bufferSize:blockSize]; [output writeStringNoTag:value]; + XCTAssertEqual(output.bytesWritten, data.length); [output flush]; + XCTAssertEqual(output.bytesWritten, data.length); actual = [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; XCTAssertEqualObjects(data, actual, @"%@", contextMessage); From 8f5e43261531ceffb7f93b46bf134b75a55a28bb Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 22 Feb 2023 12:45:42 -0800 Subject: [PATCH 077/463] Add regression test for invalid inline string code generation. A bug was found during code review that didn't trigger any test failure The fast TDP entry did not have the index in the aux_idx field. PiperOrigin-RevId: 511570831 --- .../protobuf/generated_message_tctable_impl.h | 2 +- src/google/protobuf/message_unittest.inc | 57 ++++++++++++++++++- src/google/protobuf/unittest.proto | 22 +++++-- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 1adea2d9f91a..13796053f916 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -271,7 +271,7 @@ inline void AlignFail(std::integral_constant, class PROTOBUF_EXPORT TcParser final { public: template - static constexpr const TcParseTableBase* GetTable() { + static constexpr auto GetTable() -> decltype(&T::_table_.header) { return &T::_table_.header; } diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 24e67b520970..8cfe8c1d5abc 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -62,11 +62,12 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/generated_message_reflection.h" -#include "google/protobuf/message.h" +#include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/io_win32.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/message.h" #include "google/protobuf/test_util2.h" @@ -1684,6 +1685,60 @@ class TestInputStream final : public io::ZeroCopyInputStream { size_t break_pos_; }; +template +static const internal::TcParseTableBase* GetTableIfAvailable(...) { + return nullptr; +} + +template +static const internal::TcParseTableBase* GetTableIfAvailable( + decltype(internal::TcParser::GetTable())) { + return internal::TcParser::GetTable(); +} + +TEST(MESSAGE_TEST_NAME, TestRegressionInlinedStringAuxIdxMismatchOnFastParser) { + using Proto = UNITTEST::InlinedStringIdxRegressionProto; + + auto* table = GetTableIfAvailable(nullptr); + // Only test when TDP is on, and we have these fields inlined. + if (table != nullptr && + table->fast_entry(1)->target() == internal::TcParser::FastSiS1) { + // optional string str1 = 1; + EXPECT_EQ(table->fast_entry(1)->bits.aux_idx(), 1); + // optional InlinedStringIdxRegressionProto sub = 2; + EXPECT_EQ(table->fast_entry(2)->bits.aux_idx(), 2); + // optional string str2 = 3; + // The aux_idx points to the inlined_string_idx and not the actual aux_idx. + EXPECT_EQ(table->fast_entry(3)->bits.aux_idx(), 2); + // optional string str3 = 4; + // The aux_idx points to the inlined_string_idx and not the actual aux_idx. + EXPECT_EQ(table->fast_entry(0)->bits.aux_idx(), 3); + } + + std::string encoded; + { + Proto proto; + // We use strings longer than SSO. + proto.set_str1(std::string(100, 'a')); + proto.set_str2(std::string(100, 'a')); + proto.set_str3(std::string(100, 'a')); + encoded = proto.SerializeAsString(); + } + Arena arena; + auto* proto = Arena::CreateMessage(&arena); + // We don't alter donation here, so it works even if the idx are bad. + ASSERT_TRUE(proto->ParseFromString(encoded)); + // Now we alter donation bits. str2's bit (#2) will be off, but its aux_idx + // (#3) will point to a donated string. + proto = Arena::CreateMessage(&arena); + proto->mutable_str1(); + proto->mutable_str2(); + proto->mutable_str3(); + // With the bug, this breaks the cleanup list, causing UB on arena + // destruction. + ASSERT_TRUE(proto->ParseFromString(encoded)); +} + TEST(MESSAGE_TEST_NAME, TestRepeatedStringParsers) { google::protobuf::Arena arena; diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 7ebd53f9a2ce..f6c57deb1b62 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -1549,7 +1549,7 @@ message EnumParseTester { // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; -}; +} // This message contains different kind of bool fields to exercise the different // parsers in table-drived. @@ -1566,7 +1566,7 @@ message BoolParseTester { // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; -}; +} message Int32ParseTester { optional int32 optional_int32_lowfield = 1; @@ -1581,7 +1581,7 @@ message Int32ParseTester { // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; -}; +} message Int64ParseTester { optional int64 optional_int64_lowfield = 1; @@ -1596,7 +1596,19 @@ message Int64ParseTester { // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; -}; +} + +message InlinedStringIdxRegressionProto { + // We mix data to make sure aux ids and inlined string idx do not match. + // aux_idx == inlined_string_idx == 1 + optional string str1 = 1; + // aux_idx == 2 + optional InlinedStringIdxRegressionProto sub = 2; + // aux_idx == 3, inlined_string_idx == 2 + optional string str2 = 3; + // aux_idx == 4, inlined_string_idx == 3 + optional bytes str3 = 4; +} message StringParseTester { optional string optional_string_lowfield = 1; @@ -1605,7 +1617,7 @@ message StringParseTester { repeated string repeated_string_lowfield = 2; repeated string repeated_string_midfield = 1002; repeated string repeated_string_hifield = 1000002; -}; +} message BadFieldNames{ optional int32 OptionalInt32 = 1; From d3ceff3def7b8ed88fe118cfb4766462bfeb811f Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Wed, 22 Feb 2023 13:49:28 -0800 Subject: [PATCH 078/463] Add OSGi headers to pom files. This addresses https://github.com/protocolbuffers/protobuf/issues/12017. PiperOrigin-RevId: 511587640 --- java/core/pom_template.xml | 20 ++++++++++++++++++++ java/lite/pom_template.xml | 19 +++++++++++++++++++ java/util/pom_template.xml | 19 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/java/core/pom_template.xml b/java/core/pom_template.xml index 4a30ddc86343..11bd41964f8d 100644 --- a/java/core/pom_template.xml +++ b/java/core/pom_template.xml @@ -18,4 +18,24 @@ {dependencies} + + + + + + org.apache.felix + maven-bundle-plugin + true + + + com.google.protobuf + https://developers.google.com/protocol-buffers/ + com.google.protobuf + com.google.protobuf;version=${project.version} + sun.misc;resolution:=optional,* + + + + + diff --git a/java/lite/pom_template.xml b/java/lite/pom_template.xml index fa6cfd2ae6b9..728c291a4be4 100644 --- a/java/lite/pom_template.xml +++ b/java/lite/pom_template.xml @@ -19,4 +19,23 @@ {dependencies} + + + + + org.apache.felix + maven-bundle-plugin + true + + + com.google.protobuf + https://developers.google.com/protocol-buffers/ + com.google.protobuf + com.google.protobuf;version=${project.version} + sun.misc;resolution:=optional,* + + + + + diff --git a/java/util/pom_template.xml b/java/util/pom_template.xml index f897892510b0..74ef795f1742 100644 --- a/java/util/pom_template.xml +++ b/java/util/pom_template.xml @@ -20,4 +20,23 @@ {dependencies} + + + + + org.apache.felix + maven-bundle-plugin + true + + + com.google.protobuf.util + https://developers.google.com/protocol-buffers/ + com.google.protobuf.util + com.google.protobuf.util;version=${project.version} + + + + + + From dbd5ae859343a89cb1fc5b2af9929511fbc747f9 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 22 Feb 2023 13:49:53 -0800 Subject: [PATCH 079/463] Allow AbstractProtoList to be created in an immutable state rather than using static blocks to call makeImmutable. PiperOrigin-RevId: 511587750 --- .../com/google/protobuf/AbstractProtobufList.java | 7 ++++++- .../java/com/google/protobuf/BooleanArrayList.java | 14 ++++++-------- .../java/com/google/protobuf/DoubleArrayList.java | 14 ++++++-------- .../java/com/google/protobuf/FloatArrayList.java | 14 ++++++-------- .../java/com/google/protobuf/IntArrayList.java | 12 +++++------- .../com/google/protobuf/LazyStringArrayList.java | 11 ++++++----- .../java/com/google/protobuf/LongArrayList.java | 14 ++++++-------- .../com/google/protobuf/ProtobufArrayList.java | 13 +++++-------- 8 files changed, 46 insertions(+), 53 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java b/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java index b8baaa083e84..4491977e18ad 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java @@ -52,7 +52,12 @@ abstract class AbstractProtobufList extends AbstractList implements Protob /** Constructs a mutable list by default. */ AbstractProtobufList() { - isMutable = true; + this(true); + } + + /** Constructs an immutable list for EMPTY lists */ + AbstractProtobufList(boolean isMutable) { + this.isMutable = isMutable; } @Override diff --git a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java index 451fce1e8499..bd1e9fadf7ae 100644 --- a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java @@ -45,10 +45,7 @@ final class BooleanArrayList extends AbstractProtobufList implements BooleanList, RandomAccess, PrimitiveNonBoxingCollection { - private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList(new boolean[0], 0); - static { - EMPTY_LIST.makeImmutable(); - } + private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList(new boolean[0], 0, false); public static BooleanArrayList emptyList() { return EMPTY_LIST; @@ -65,15 +62,16 @@ public static BooleanArrayList emptyList() { /** Constructs a new mutable {@code BooleanArrayList} with default capacity. */ BooleanArrayList() { - this(new boolean[DEFAULT_CAPACITY], 0); + this(new boolean[DEFAULT_CAPACITY], 0, true); } /** * Constructs a new mutable {@code BooleanArrayList} containing the same elements as {@code * other}. */ - private BooleanArrayList(boolean[] other, int size) { - array = other; + private BooleanArrayList(boolean[] other, int size, boolean isMutable) { + super(isMutable); + this.array = other; this.size = size; } @@ -126,7 +124,7 @@ public BooleanList mutableCopyWithCapacity(int capacity) { if (capacity < size) { throw new IllegalArgumentException(); } - return new BooleanArrayList(Arrays.copyOf(array, capacity), size); + return new BooleanArrayList(Arrays.copyOf(array, capacity), size, true); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java index 408565343720..40a3e79b8f84 100644 --- a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java @@ -45,10 +45,7 @@ final class DoubleArrayList extends AbstractProtobufList implements DoubleList, RandomAccess, PrimitiveNonBoxingCollection { - private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList(new double[0], 0); - static { - EMPTY_LIST.makeImmutable(); - } + private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList(new double[0], 0, false); public static DoubleArrayList emptyList() { return EMPTY_LIST; @@ -65,14 +62,15 @@ public static DoubleArrayList emptyList() { /** Constructs a new mutable {@code DoubleArrayList} with default capacity. */ DoubleArrayList() { - this(new double[DEFAULT_CAPACITY], 0); + this(new double[DEFAULT_CAPACITY], 0, true); } /** * Constructs a new mutable {@code DoubleArrayList} containing the same elements as {@code other}. */ - private DoubleArrayList(double[] other, int size) { - array = other; + private DoubleArrayList(double[] other, int size, boolean isMutable) { + super(isMutable); + this.array = other; this.size = size; } @@ -126,7 +124,7 @@ public DoubleList mutableCopyWithCapacity(int capacity) { if (capacity < size) { throw new IllegalArgumentException(); } - return new DoubleArrayList(Arrays.copyOf(array, capacity), size); + return new DoubleArrayList(Arrays.copyOf(array, capacity), size, true); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/FloatArrayList.java b/java/core/src/main/java/com/google/protobuf/FloatArrayList.java index e6feba8a354c..023fd6cff80d 100644 --- a/java/core/src/main/java/com/google/protobuf/FloatArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/FloatArrayList.java @@ -45,10 +45,7 @@ final class FloatArrayList extends AbstractProtobufList implements FloatList, RandomAccess, PrimitiveNonBoxingCollection { - private static final FloatArrayList EMPTY_LIST = new FloatArrayList(new float[0], 0); - static { - EMPTY_LIST.makeImmutable(); - } + private static final FloatArrayList EMPTY_LIST = new FloatArrayList(new float[0], 0, false); public static FloatArrayList emptyList() { return EMPTY_LIST; @@ -65,14 +62,15 @@ public static FloatArrayList emptyList() { /** Constructs a new mutable {@code FloatArrayList} with default capacity. */ FloatArrayList() { - this(new float[DEFAULT_CAPACITY], 0); + this(new float[DEFAULT_CAPACITY], 0, true); } /** * Constructs a new mutable {@code FloatArrayList} containing the same elements as {@code other}. */ - private FloatArrayList(float[] other, int size) { - array = other; + private FloatArrayList(float[] other, int size, boolean isMutable) { + super(isMutable); + this.array = other; this.size = size; } @@ -125,7 +123,7 @@ public FloatList mutableCopyWithCapacity(int capacity) { if (capacity < size) { throw new IllegalArgumentException(); } - return new FloatArrayList(Arrays.copyOf(array, capacity), size); + return new FloatArrayList(Arrays.copyOf(array, capacity), size, true); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/IntArrayList.java b/java/core/src/main/java/com/google/protobuf/IntArrayList.java index 9daeebed9923..692da9c12c7f 100644 --- a/java/core/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/IntArrayList.java @@ -45,10 +45,7 @@ final class IntArrayList extends AbstractProtobufList implements IntList, RandomAccess, PrimitiveNonBoxingCollection { - private static final IntArrayList EMPTY_LIST = new IntArrayList(new int[0], 0); - static { - EMPTY_LIST.makeImmutable(); - } + private static final IntArrayList EMPTY_LIST = new IntArrayList(new int[0], 0, false); public static IntArrayList emptyList() { return EMPTY_LIST; @@ -65,13 +62,14 @@ public static IntArrayList emptyList() { /** Constructs a new mutable {@code IntArrayList} with default capacity. */ IntArrayList() { - this(new int[DEFAULT_CAPACITY], 0); + this(new int[DEFAULT_CAPACITY], 0, true); } /** * Constructs a new mutable {@code IntArrayList} containing the same elements as {@code other}. */ - private IntArrayList(int[] other, int size) { + private IntArrayList(int[] other, int size, boolean isMutable) { + super(isMutable); array = other; this.size = size; } @@ -125,7 +123,7 @@ public IntList mutableCopyWithCapacity(int capacity) { if (capacity < size) { throw new IllegalArgumentException(); } - return new IntArrayList(Arrays.copyOf(array, capacity), size); + return new IntArrayList(Arrays.copyOf(array, capacity), size, true); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java b/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java index 7ef968738d09..3ff5a3bba9d6 100644 --- a/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java @@ -61,11 +61,7 @@ public class LazyStringArrayList extends AbstractProtobufList implements LazyStringList, RandomAccess { - private static final LazyStringArrayList EMPTY_LIST = new LazyStringArrayList(); - - static { - EMPTY_LIST.makeImmutable(); - } + private static final LazyStringArrayList EMPTY_LIST = new LazyStringArrayList(false); /** Returns an empty immutable {@code LazyStringArrayList} instance */ public static LazyStringArrayList emptyList() { @@ -87,6 +83,11 @@ public LazyStringArrayList() { this(DEFAULT_CAPACITY); } + private LazyStringArrayList(boolean isMutable) { + super(isMutable); + this.list = Collections.emptyList(); + } + public LazyStringArrayList(int initialCapacity) { this(new ArrayList(initialCapacity)); } diff --git a/java/core/src/main/java/com/google/protobuf/LongArrayList.java b/java/core/src/main/java/com/google/protobuf/LongArrayList.java index bda43a41bb4a..b5ca4decf4b6 100644 --- a/java/core/src/main/java/com/google/protobuf/LongArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/LongArrayList.java @@ -45,10 +45,7 @@ final class LongArrayList extends AbstractProtobufList implements LongList, RandomAccess, PrimitiveNonBoxingCollection { - private static final LongArrayList EMPTY_LIST = new LongArrayList(new long[0], 0); - static { - EMPTY_LIST.makeImmutable(); - } + private static final LongArrayList EMPTY_LIST = new LongArrayList(new long[0], 0, false); public static LongArrayList emptyList() { return EMPTY_LIST; @@ -65,14 +62,15 @@ public static LongArrayList emptyList() { /** Constructs a new mutable {@code LongArrayList} with default capacity. */ LongArrayList() { - this(new long[DEFAULT_CAPACITY], 0); + this(new long[DEFAULT_CAPACITY], 0, true); } /** * Constructs a new mutable {@code LongArrayList} containing the same elements as {@code other}. */ - private LongArrayList(long[] other, int size) { - array = other; + private LongArrayList(long[] other, int size, boolean isMutable) { + super(isMutable); + this.array = other; this.size = size; } @@ -125,7 +123,7 @@ public LongList mutableCopyWithCapacity(int capacity) { if (capacity < size) { throw new IllegalArgumentException(); } - return new LongArrayList(Arrays.copyOf(array, capacity), size); + return new LongArrayList(Arrays.copyOf(array, capacity), size, true); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java b/java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java index 33e4bd5e35e2..28b547c927c9 100644 --- a/java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java @@ -38,11 +38,7 @@ final class ProtobufArrayList extends AbstractProtobufList implements RandomAccess { private static final ProtobufArrayList EMPTY_LIST = - new ProtobufArrayList(new Object[0], 0); - - static { - EMPTY_LIST.makeImmutable(); - } + new ProtobufArrayList(new Object[0], 0, false); @SuppressWarnings("unchecked") // Guaranteed safe by runtime. public static ProtobufArrayList emptyList() { @@ -54,10 +50,11 @@ public static ProtobufArrayList emptyList() { @SuppressWarnings("unchecked") ProtobufArrayList() { - this((E[]) new Object[DEFAULT_CAPACITY], 0); + this((E[]) new Object[DEFAULT_CAPACITY], 0, true); } - private ProtobufArrayList(E[] array, int size) { + private ProtobufArrayList(E[] array, int size, boolean isMutable) { + super(isMutable); this.array = array; this.size = size; } @@ -70,7 +67,7 @@ public ProtobufArrayList mutableCopyWithCapacity(int capacity) { E[] newArray = Arrays.copyOf(array, capacity); - return new ProtobufArrayList(newArray, size); + return new ProtobufArrayList(newArray, size, true); } @Override From fa82155c653776304bf6d03c33bea744db1b5eff Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 22 Feb 2023 14:29:49 -0800 Subject: [PATCH 080/463] Fix mutability bug in Java proto lite: sub-messages inside of oneofs were not being set as immutable. This would allow code to get a builder for a sub-message and modify the original (supposedly immutable) copy. PiperOrigin-RevId: 511598810 --- .../com/google/protobuf/MessageSchema.java | 6 +++++ .../java/com/google/protobuf/LiteTest.java | 24 +++++++++++++------ src/google/protobuf/unittest_lite.proto | 1 + 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/MessageSchema.java b/java/core/src/main/java/com/google/protobuf/MessageSchema.java index 084d687d1a69..ae5dddc53bb2 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageSchema.java +++ b/java/core/src/main/java/com/google/protobuf/MessageSchema.java @@ -5510,6 +5510,12 @@ public void makeImmutable(T message) { getMessageFieldSchema(pos).makeImmutable(UNSAFE.getObject(message, offset)); } break; + case 60: // ONEOF_MESSAGE + case 68: // ONEOF_GROUP + if (isOneofPresent(message, numberAt(pos), pos)) { + getMessageFieldSchema(pos).makeImmutable(UNSAFE.getObject(message, offset)); + } + break; case 18: // DOUBLE_LIST: case 19: // FLOAT_LIST: case 20: // INT64_LIST: diff --git a/java/lite/src/test/java/com/google/protobuf/LiteTest.java b/java/lite/src/test/java/com/google/protobuf/LiteTest.java index a4f95a2666de..a58ce95df060 100644 --- a/java/lite/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/lite/src/test/java/com/google/protobuf/LiteTest.java @@ -190,6 +190,18 @@ public void testAddAll() { } } + @Test + public void testParsedOneofSubMessageIsImmutable() throws InvalidProtocolBufferException { + TestAllTypesLite message = + TestAllTypesLite.parseFrom( + TestAllTypesLite.newBuilder() + .setOneofNestedMessage(NestedMessage.newBuilder().addDd(1234).build()) + .build() + .toByteArray()); + IntArrayList subList = (IntArrayList) message.getOneofNestedMessage().getDdList(); + assertThat(subList.isModifiable()).isFalse(); + } + @Test public void testMemoization() throws Exception { GeneratedMessageLite message = TestUtilLite.getAllLiteExtensionsSet(); @@ -2365,8 +2377,7 @@ public void testEqualsAndHashCodeWithUnknownFields() throws InvalidProtocolBuffe Foo fooWithOnlyValue = Foo.newBuilder().setValue(1).build(); Foo fooWithValueAndExtension = - fooWithOnlyValue - .toBuilder() + fooWithOnlyValue.toBuilder() .setValue(1) .setExtension(Bar.fooExt, Bar.newBuilder().setName("name").build()) .build(); @@ -2382,8 +2393,7 @@ public void testEqualsAndHashCodeWithExtensions() throws InvalidProtocolBufferEx Foo fooWithOnlyValue = Foo.newBuilder().setValue(1).build(); Foo fooWithValueAndExtension = - fooWithOnlyValue - .toBuilder() + fooWithOnlyValue.toBuilder() .setValue(1) .setExtension(Bar.fooExt, Bar.newBuilder().setName("name").build()) .build(); @@ -2515,9 +2525,9 @@ public void testParseFromByteBufferThrows_extensions() { assertWithMessage("expected exception").fail(); } catch (InvalidProtocolBufferException expected) { assertThat( - TestAllExtensionsLite.newBuilder() - .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) - .build()) + TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .build()) .isEqualTo(expected.getUnfinishedMessage()); } } diff --git a/src/google/protobuf/unittest_lite.proto b/src/google/protobuf/unittest_lite.proto index 010d4a9c0b21..e2730c64196d 100644 --- a/src/google/protobuf/unittest_lite.proto +++ b/src/google/protobuf/unittest_lite.proto @@ -47,6 +47,7 @@ message TestAllTypesLite { message NestedMessage { optional int32 bb = 1; optional int64 cc = 2; + repeated int32 dd = 3 [packed = true]; } message NestedMessage2 { From 684a3dedf66d3c0979e1a99b4880d228e6569b51 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Thu, 23 Feb 2023 08:22:14 -0800 Subject: [PATCH 081/463] Migrate away from examining the syntax of a proto file via reflection. PiperOrigin-RevId: 511792205 --- .../main/java/com/google/protobuf/DynamicMessage.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java index 738c30392d89..e564a6357696 100644 --- a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java @@ -535,11 +535,10 @@ public Builder setField(FieldDescriptor field, Object value) { fields.clearField(oldField); } oneofCases[index] = field; - } else if (field.getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3) { - if (!field.isRepeated() - && field.getJavaType() != FieldDescriptor.JavaType.MESSAGE - && value.equals(field.getDefaultValue())) { - // In proto3, setting a field to its default value is equivalent to clearing the field. + } else if (!field.hasPresence()) { + if (!field.isRepeated() && value.equals(field.getDefaultValue())) { + // Setting a field without presence to its default value is equivalent to clearing the + // field. fields.clearField(field); return this; } From 2a55706f9d36afd1742c97fd0d611ea8af0ac3f8 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Thu, 23 Feb 2023 08:40:01 -0800 Subject: [PATCH 082/463] Migrate away from examining the syntax of a proto file via reflection. PiperOrigin-RevId: 511796130 --- .../main/java/com/google/protobuf/GeneratedMessageV3.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index b25011e0d9ff..b5d3c76c5fb3 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -2407,10 +2407,7 @@ public void clear(final GeneratedMessageV3.Builder builder) { isOneofField = descriptor.getContainingOneof() != null && !descriptor.getContainingOneof().isSynthetic(); - hasHasMethod = - descriptor.getFile().getSyntax() == FileDescriptor.Syntax.PROTO2 - || descriptor.hasOptionalKeyword() - || (!isOneofField && descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE); + hasHasMethod = descriptor.hasPresence(); ReflectionInvoker reflectionInvoker = new ReflectionInvoker( descriptor, From 541914157e7e3cd5baf47d1304e78727975a6cf0 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Thu, 23 Feb 2023 09:21:41 -0800 Subject: [PATCH 083/463] Migrate away from examining the syntax of a proto file via reflection. PiperOrigin-RevId: 511806036 --- .../src/main/java/com/google/protobuf/util/JsonFormat.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 96bc38898f2d..719980a6a24e 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1925,10 +1925,10 @@ private EnumValueDescriptor parseEnum(EnumDescriptor enumDescriptor, JsonElement // Try to interpret the value as a number. try { int numericValue = parseInt32(json); - if (enumDescriptor.getFile().getSyntax() == FileDescriptor.Syntax.PROTO3) { - result = enumDescriptor.findValueByNumberCreatingIfUnknown(numericValue); - } else { + if (enumDescriptor.isClosed()) { result = enumDescriptor.findValueByNumber(numericValue); + } else { + result = enumDescriptor.findValueByNumberCreatingIfUnknown(numericValue); } } catch (InvalidProtocolBufferException e) { // Fall through. This exception is about invalid int32 value we get from parseInt32() but From d40aadf823cf7e1e62b65561656f689da8969463 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 23 Feb 2023 09:29:42 -0800 Subject: [PATCH 084/463] Add casts to make protobuf compatible with Java 1.8 runtime. Fix for: https://github.com/protocolbuffers/protobuf/issues/11393 PiperOrigin-RevId: 511807920 --- java/core/BUILD.bazel | 1 + .../com/google/protobuf/AllocatedBuffer.java | 2 +- .../com/google/protobuf/BinaryWriter.java | 28 ++++---- .../com/google/protobuf/ByteBufferWriter.java | 2 +- .../google/protobuf/CodedOutputStream.java | 28 ++++---- .../java/com/google/protobuf/Internal.java | 8 ++- .../IterableByteBufferInputStream.java | 4 +- .../google/protobuf/Java8Compatibility.java | 67 +++++++++++++++++++ .../com/google/protobuf/NioByteString.java | 11 ++- .../java/com/google/protobuf/TextFormat.java | 2 +- .../main/java/com/google/protobuf/Utf8.java | 10 +-- java/lite/pom.xml | 1 + 12 files changed, 118 insertions(+), 46 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/Java8Compatibility.java diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 6113169c6c7a..c0e52a4ff4bc 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -49,6 +49,7 @@ LITE_SRCS = [ "src/main/java/com/google/protobuf/Internal.java", "src/main/java/com/google/protobuf/InvalidProtocolBufferException.java", "src/main/java/com/google/protobuf/IterableByteBufferInputStream.java", + "src/main/java/com/google/protobuf/Java8Compatibility.java", "src/main/java/com/google/protobuf/JavaType.java", "src/main/java/com/google/protobuf/LazyField.java", "src/main/java/com/google/protobuf/LazyFieldLite.java", diff --git a/java/core/src/main/java/com/google/protobuf/AllocatedBuffer.java b/java/core/src/main/java/com/google/protobuf/AllocatedBuffer.java index 94b09944d1a8..75de57a32ed0 100644 --- a/java/core/src/main/java/com/google/protobuf/AllocatedBuffer.java +++ b/java/core/src/main/java/com/google/protobuf/AllocatedBuffer.java @@ -189,7 +189,7 @@ public int position() { @Override public AllocatedBuffer position(int position) { - buffer.position(position); + Java8Compatibility.position(buffer, position); return this; } diff --git a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java index 66cf51d24c4a..7eb7886312ce 100644 --- a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java +++ b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java @@ -2019,8 +2019,8 @@ private void nextBuffer(AllocatedBuffer allocatedBuffer) { buffers.addFirst(allocatedBuffer); buffer = nioBuffer; - buffer.limit(buffer.capacity()); - buffer.position(0); + Java8Compatibility.limit(buffer, buffer.capacity()); + Java8Compatibility.position(buffer, 0); // Set byte order to little endian for fast writing of fixed 32/64. buffer.order(ByteOrder.LITTLE_ENDIAN); @@ -2046,7 +2046,7 @@ void finishCurrentBuffer() { if (buffer != null) { totalDoneBytes += bytesWrittenToCurrentBuffer(); // Update the indices on the netty buffer. - buffer.position(pos + 1); + Java8Compatibility.position(buffer, pos + 1); buffer = null; pos = 0; limitMinusOne = 0; @@ -2475,7 +2475,7 @@ public void write(byte[] value, int offset, int length) { } pos -= length; - buffer.position(pos + 1); + Java8Compatibility.position(buffer, pos + 1); buffer.put(value, offset, length); } @@ -2494,7 +2494,7 @@ public void writeLazy(byte[] value, int offset, int length) { } pos -= length; - buffer.position(pos + 1); + Java8Compatibility.position(buffer, pos + 1); buffer.put(value, offset, length); } @@ -2506,7 +2506,7 @@ public void write(ByteBuffer value) { } pos -= length; - buffer.position(pos + 1); + Java8Compatibility.position(buffer, pos + 1); buffer.put(value); } @@ -2526,7 +2526,7 @@ public void writeLazy(ByteBuffer value) { } pos -= length; - buffer.position(pos + 1); + Java8Compatibility.position(buffer, pos + 1); buffer.put(value); } @@ -2576,8 +2576,8 @@ private void nextBuffer(AllocatedBuffer allocatedBuffer) { buffers.addFirst(allocatedBuffer); buffer = nioBuffer; - buffer.limit(buffer.capacity()); - buffer.position(0); + Java8Compatibility.limit(buffer, buffer.capacity()); + Java8Compatibility.position(buffer, 0); bufferOffset = UnsafeUtil.addressOffset(buffer); limitMinusOne = bufferOffset + (buffer.limit() - 1); @@ -2602,7 +2602,7 @@ void finishCurrentBuffer() { if (buffer != null) { totalDoneBytes += bytesWrittenToCurrentBuffer(); // Update the indices on the netty buffer. - buffer.position(bufferPos() + 1); + Java8Compatibility.position(buffer, bufferPos() + 1); buffer = null; pos = 0; limitMinusOne = 0; @@ -3016,7 +3016,7 @@ public void write(byte[] value, int offset, int length) { } pos -= length; - buffer.position(bufferPos() + 1); + Java8Compatibility.position(buffer, bufferPos() + 1); buffer.put(value, offset, length); } @@ -3035,7 +3035,7 @@ public void writeLazy(byte[] value, int offset, int length) { } pos -= length; - buffer.position(bufferPos() + 1); + Java8Compatibility.position(buffer, bufferPos() + 1); buffer.put(value, offset, length); } @@ -3047,7 +3047,7 @@ public void write(ByteBuffer value) { } pos -= length; - buffer.position(bufferPos() + 1); + Java8Compatibility.position(buffer, bufferPos() + 1); buffer.put(value); } @@ -3067,7 +3067,7 @@ public void writeLazy(ByteBuffer value) { } pos -= length; - buffer.position(bufferPos() + 1); + Java8Compatibility.position(buffer, bufferPos() + 1); buffer.put(value); } diff --git a/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java b/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java index 2cb3ada67fb6..3970b0ea5b85 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java +++ b/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java @@ -107,7 +107,7 @@ static void write(ByteBuffer buffer, OutputStream output) throws IOException { } } finally { // Restore the initial position. - buffer.position(initialPos); + Java8Compatibility.position(buffer, initialPos); } } diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index 4ad83099d769..44dd4dc20ebd 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -1271,7 +1271,7 @@ public final void writeRawBytes(final ByteBuffer value) throws IOException { write(value.array(), value.arrayOffset(), value.capacity()); } else { ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); write(duplicated); } } @@ -1522,7 +1522,7 @@ private static final class HeapNioEncoder extends ArrayEncoder { @Override public void flush() { // Update the position on the buffer. - byteBuffer.position(initialPosition + getTotalBytesWritten()); + Java8Compatibility.position(byteBuffer, initialPosition + getTotalBytesWritten()); } } @@ -1684,7 +1684,7 @@ public void writeRawBytes(final ByteBuffer value) throws IOException { write(value.array(), value.arrayOffset(), value.capacity()); } else { ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); write(duplicated); } } @@ -1794,18 +1794,18 @@ public void writeStringNoTag(String value) throws IOException { // Save the current position and increment past the length field. We'll come back // and write the length field after the encoding is complete. final int startOfBytes = buffer.position() + minLengthVarIntSize; - buffer.position(startOfBytes); + Java8Compatibility.position(buffer, startOfBytes); // Encode the string. encode(value); // Now go back to the beginning and write the length. int endOfBytes = buffer.position(); - buffer.position(startPos); + Java8Compatibility.position(buffer, startPos); writeUInt32NoTag(endOfBytes - startOfBytes); // Reposition the buffer past the written data. - buffer.position(endOfBytes); + Java8Compatibility.position(buffer, endOfBytes); } else { final int length = Utf8.encodedLength(value); writeUInt32NoTag(length); @@ -1813,7 +1813,7 @@ public void writeStringNoTag(String value) throws IOException { } } catch (UnpairedSurrogateException e) { // Roll back the change and convert to an IOException. - buffer.position(startPos); + Java8Compatibility.position(buffer, startPos); // TODO(nathanmittler): We should throw an IOException here instead. inefficientWriteStringNoTag(value, e); @@ -1826,7 +1826,7 @@ public void writeStringNoTag(String value) throws IOException { @Override public void flush() { // Update the position of the original buffer. - originalBuffer.position(buffer.position()); + Java8Compatibility.position(originalBuffer, buffer.position()); } @Override @@ -2014,7 +2014,7 @@ public void writeRawBytes(ByteBuffer value) throws IOException { write(value.array(), value.arrayOffset(), value.capacity()); } else { ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); write(duplicated); } } @@ -2150,7 +2150,7 @@ public void writeStringNoTag(String value) throws IOException { // Save the current position and increment past the length field. We'll come back // and write the length field after the encoding is complete. int stringStart = bufferPos(position) + minLengthVarIntSize; - buffer.position(stringStart); + Java8Compatibility.position(buffer, stringStart); // Encode the string. Utf8.encodeUtf8(value, buffer); @@ -2187,7 +2187,7 @@ public void writeStringNoTag(String value) throws IOException { @Override public void flush() { // Update the position of the original buffer. - originalBuffer.position(bufferPos(position)); + Java8Compatibility.position(originalBuffer, bufferPos(position)); } @Override @@ -2201,7 +2201,7 @@ public int getTotalBytesWritten() { } private void repositionBuffer(long pos) { - buffer.position(bufferPos(pos)); + Java8Compatibility.position(buffer, bufferPos(pos)); } private int bufferPos(long pos) { @@ -2478,7 +2478,7 @@ public void writeRawBytes(final ByteBuffer value) throws IOException { write(value.array(), value.arrayOffset(), value.capacity()); } else { ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); write(duplicated); } } @@ -2792,7 +2792,7 @@ public void writeRawBytes(final ByteBuffer value) throws IOException { write(value.array(), value.arrayOffset(), value.capacity()); } else { ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); write(duplicated); } } diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index 7a7270b8c3f1..127a16566edf 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -313,7 +313,11 @@ public static boolean equalsByteBuffer(ByteBuffer a, ByteBuffer b) { } // ByteBuffer.equals() will only compare the remaining bytes, but we want to // compare all the content. - return a.duplicate().clear().equals(b.duplicate().clear()); + ByteBuffer aDuplicate = a.duplicate(); + Java8Compatibility.clear(aDuplicate); + ByteBuffer bDuplicate = b.duplicate(); + Java8Compatibility.clear(bDuplicate); + return aDuplicate.equals(bDuplicate); } /** Helper method for implementing {@link Message#equals(Object)} for bytes field. */ @@ -353,7 +357,7 @@ public static int hashCodeByteBuffer(ByteBuffer bytes) { bytes.capacity() > DEFAULT_BUFFER_SIZE ? DEFAULT_BUFFER_SIZE : bytes.capacity(); final byte[] buffer = new byte[bufferSize]; final ByteBuffer duplicated = bytes.duplicate(); - duplicated.clear(); + Java8Compatibility.clear(duplicated); int h = bytes.capacity(); while (duplicated.remaining() > 0) { final int length = diff --git a/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java index 713e80646931..1e571cfbbb54 100644 --- a/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java @@ -140,9 +140,9 @@ public int read(byte[] output, int offset, int length) throws IOException { updateCurrentByteBufferPos(length); } else { int prevPos = currentByteBuffer.position(); - currentByteBuffer.position(currentByteBufferPos); + Java8Compatibility.position(currentByteBuffer, currentByteBufferPos); currentByteBuffer.get(output, offset, length); - currentByteBuffer.position(prevPos); + Java8Compatibility.position(currentByteBuffer, prevPos); updateCurrentByteBufferPos(length); } return length; diff --git a/java/core/src/main/java/com/google/protobuf/Java8Compatibility.java b/java/core/src/main/java/com/google/protobuf/Java8Compatibility.java new file mode 100644 index 000000000000..ef181c2ae55b --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/Java8Compatibility.java @@ -0,0 +1,67 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.nio.Buffer; + +/** + * Wrappers around {@link Buffer} methods that are covariantly overridden in Java 9+. See + * https://github.com/protocolbuffers/protobuf/issues/11393 + * + *

TODO(b/270454719) remove when Java 8 support is no longer needed. + */ +final class Java8Compatibility { + static void clear(Buffer b) { + b.clear(); + } + + static void flip(Buffer b) { + b.flip(); + } + + static void limit(Buffer b, int limit) { + b.limit(limit); + } + + static void mark(Buffer b) { + b.mark(); + } + + static void position(Buffer b, int position) { + b.position(position); + } + + static void reset(Buffer b) { + b.reset(); + } + + private Java8Compatibility() {} +} diff --git a/java/core/src/main/java/com/google/protobuf/NioByteString.java b/java/core/src/main/java/com/google/protobuf/NioByteString.java index 1e594ff878c5..f625edafb9ef 100644 --- a/java/core/src/main/java/com/google/protobuf/NioByteString.java +++ b/java/core/src/main/java/com/google/protobuf/NioByteString.java @@ -37,7 +37,6 @@ import java.io.InvalidObjectException; import java.io.ObjectInputStream; import java.io.OutputStream; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.InvalidMarkException; @@ -110,7 +109,7 @@ public ByteString substring(int beginIndex, int endIndex) { protected void copyToInternal( byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { ByteBuffer slice = buffer.slice(); - ((Buffer) slice).position(sourceOffset); + Java8Compatibility.position(slice, sourceOffset); slice.get(target, targetOffset, numberToCopy); } @@ -224,7 +223,7 @@ public InputStream newInput() { @Override public void mark(int readlimit) { - buf.mark(); + Java8Compatibility.mark(buf); } @Override @@ -235,7 +234,7 @@ public boolean markSupported() { @Override public void reset() throws IOException { try { - buf.reset(); + Java8Compatibility.reset(buf); } catch (InvalidMarkException e) { throw new IOException(e); } @@ -286,8 +285,8 @@ private ByteBuffer slice(int beginIndex, int endIndex) { } ByteBuffer slice = buffer.slice(); - ((Buffer) slice).position(beginIndex - buffer.position()); - ((Buffer) slice).limit(endIndex - buffer.position()); + Java8Compatibility.position(slice, beginIndex - buffer.position()); + Java8Compatibility.limit(slice, endIndex - buffer.position()); return slice; } } diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 8c3ac5caf10b..9d28d06c9527 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -1710,7 +1710,7 @@ private static StringBuilder toStringBuilder(final Readable input) throws IOExce if (n == -1) { break; } - buffer.flip(); + Java8Compatibility.flip(buffer); text.append(buffer, 0, n); } return text; diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index c74497cd816a..7c6823d91ffa 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -770,7 +770,7 @@ final void encodeUtf8(CharSequence in, ByteBuffer out) { if (out.hasArray()) { final int offset = out.arrayOffset(); int endIndex = Utf8.encode(in, out.array(), offset + out.position(), out.remaining()); - out.position(endIndex - offset); + Java8Compatibility.position(out, endIndex - offset); } else if (out.isDirect()) { encodeUtf8Direct(in, out); } else { @@ -801,7 +801,7 @@ final void encodeUtf8Default(CharSequence in, ByteBuffer out) { } if (inIx == inLength) { // Successfully encoded the entire string. - out.position(outIx + inIx); + Java8Compatibility.position(out, outIx + inIx); return; } @@ -844,7 +844,7 @@ final void encodeUtf8Default(CharSequence in, ByteBuffer out) { } // Successfully encoded the entire string. - out.position(outIx); + Java8Compatibility.position(out, outIx); } catch (IndexOutOfBoundsException e) { // TODO(nathanmittler): Consider making the API throw IndexOutOfBoundsException instead. @@ -1545,7 +1545,7 @@ void encodeUtf8Direct(CharSequence in, ByteBuffer out) { } if (inIx == inLimit) { // We're done, it was ASCII encoded. - out.position((int) (outIx - address)); + Java8Compatibility.position(out, (int) (outIx - address)); return; } @@ -1585,7 +1585,7 @@ void encodeUtf8Direct(CharSequence in, ByteBuffer out) { } // All bytes have been encoded. - out.position((int) (outIx - address)); + Java8Compatibility.position(out, (int) (outIx - address)); } /** diff --git a/java/lite/pom.xml b/java/lite/pom.xml index af34943dabf7..b59c48a9eade 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -125,6 +125,7 @@ Internal.java InvalidProtocolBufferException.java IterableByteBufferInputStream.java + Java8Compatibility.java JavaType.java LazyField.java LazyFieldLite.java From cfea0399181b2b213aa65cf3d28d3ac523a6c5ed Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 23 Feb 2023 11:07:18 -0800 Subject: [PATCH 085/463] Automated rollback of commit 47b5f81c7f944b23a59b6ee9b0f0d2e9a517e10b. PiperOrigin-RevId: 511834418 --- src/google/protobuf/port_def.inc | 4 ---- src/google/protobuf/repeated_ptr_field.h | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 5935e7f45a65..3283030c16dd 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -213,10 +213,6 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // Owner: shaod@, gberg@ #define PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL 1 -// Used to remove `RepeatedPtrField::GetArena() const`. -// Owner: ezb@ -#define PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API 1 - #endif #ifdef PROTOBUF_VERSION diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index 2265cf5dccb5..bb5ed22ea511 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -1185,10 +1185,8 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase { // Gets the arena on which this RepeatedPtrField stores its elements. inline Arena* GetArena(); -#ifndef PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API ABSL_DEPRECATED("This will be removed in a future release") inline Arena* GetArena() const; -#endif // !PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API // For internal use only. // @@ -1554,12 +1552,10 @@ inline Arena* RepeatedPtrField::GetArena() { return RepeatedPtrFieldBase::GetArena(); } -#ifndef PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API template inline Arena* RepeatedPtrField::GetArena() const { return RepeatedPtrFieldBase::GetArena(); } -#endif // !PROTOBUF_FUTURE_REMOVE_CONST_REPEATEDFIELD_GETARENA_API template inline Arena* RepeatedPtrField::GetOwningArena() const { From f79cae2a8e1ab2da84fcdc3aacea43e2438d9cab Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Thu, 23 Feb 2023 12:41:50 -0800 Subject: [PATCH 086/463] Add TcParseFunction enumeration PiperOrigin-RevId: 511859491 --- .../protobuf/generated_message_tctable_gen.cc | 229 +++++++++++------- .../protobuf/generated_message_tctable_impl.h | 96 ++++++++ 2 files changed, 241 insertions(+), 84 deletions(-) diff --git a/src/google/protobuf/generated_message_tctable_gen.cc b/src/google/protobuf/generated_message_tctable_gen.cc index a5791f92178a..c5f8e195b191 100644 --- a/src/google/protobuf/generated_message_tctable_gen.cc +++ b/src/google/protobuf/generated_message_tctable_gen.cc @@ -78,100 +78,161 @@ bool GetEnumValidationRange(const EnumDescriptor* enum_type, int16_t& start, } } +absl::string_view ParseFunctionValue(TcParseFunction function) { +#define PROTOBUF_TC_PARSE_FUNCTION_X(value) #value, + static constexpr absl::string_view functions[] = { + {}, PROTOBUF_TC_PARSE_FUNCTION_LIST}; +#undef PROTOBUF_TC_PARSE_FUNCTION_X + return functions[static_cast(function)]; +}; + +enum class EnumRangeInfo { + kNone, // No contiguous range + kContiguous, // Has a contiguous range + kContiguous0, // Has a small contiguous range starting at 0 + kContiguous1, // Has a small contiguous range starting at 1 +}; + +// Returns enum validation range info, and sets `rmax_value` iff +// the returned range is a small range. `rmax_value` is guaranteed +// to remain unchanged if the enum range is not small. +EnumRangeInfo GetEnumRangeInfo(const FieldDescriptor* field, + uint8_t& rmax_value) { + int16_t start; + uint16_t size; + if (!GetEnumValidationRange(field->enum_type(), start, size)) { + return EnumRangeInfo::kNone; + } + int max_value = start + size - 1; + if (max_value <= 127 && (start == 0 || start == 1)) { + rmax_value = static_cast(max_value); + return start == 0 ? EnumRangeInfo::kContiguous0 + : EnumRangeInfo::kContiguous1; + } + return EnumRangeInfo::kContiguous; +} + void PopulateFastFieldEntry(const TailCallTableInfo::FieldEntryInfo& entry, const TailCallTableInfo::PerFieldOptions& options, TailCallTableInfo::FastFieldInfo& info) { +#define PROTOBUF_PICK_FUNCTION(fn) \ + (field->number() < 16 ? TcParseFunction::fn##1 : TcParseFunction::fn##2) + +#define PROTOBUF_PICK_SINGLE_FUNCTION(fn) PROTOBUF_PICK_FUNCTION(fn##S) + +#define PROTOBUF_PICK_REPEATABLE_FUNCTION(fn) \ + (field->is_repeated() ? PROTOBUF_PICK_FUNCTION(fn##R) \ + : PROTOBUF_PICK_FUNCTION(fn##S)) + +#define PROTOBUF_PICK_PACKABLE_FUNCTION(fn) \ + (field->is_packed() ? PROTOBUF_PICK_FUNCTION(fn##P) \ + : field->is_repeated() ? PROTOBUF_PICK_FUNCTION(fn##R) \ + : PROTOBUF_PICK_FUNCTION(fn##S)) + +#define PROTOBUF_PICK_STRING_FUNCTION(fn) \ + (field->options().ctype() == FieldOptions::CORD \ + ? PROTOBUF_PICK_FUNCTION(fn##cS) \ + : options.is_string_inlined ? PROTOBUF_PICK_FUNCTION(fn##iS) \ + : PROTOBUF_PICK_REPEATABLE_FUNCTION(fn)) + +#define PROTOBUF_PICK_MESSAGE_FUNCTION(fn) \ + (options.use_direct_tcparser_table \ + ? PROTOBUF_PICK_REPEATABLE_FUNCTION(fn##t) \ + : PROTOBUF_PICK_REPEATABLE_FUNCTION(fn##d)) + const FieldDescriptor* field = entry.field; - std::string name = "::_pbi::TcParser::Fast"; - uint8_t aux_idx = static_cast(entry.aux_idx); - - static const char* kPrefix[] = { - nullptr, // 0 - "F64", // TYPE_DOUBLE = 1, - "F32", // TYPE_FLOAT = 2, - "V64", // TYPE_INT64 = 3, - "V64", // TYPE_UINT64 = 4, - "V32", // TYPE_INT32 = 5, - "F64", // TYPE_FIXED64 = 6, - "F32", // TYPE_FIXED32 = 7, - "V8", // TYPE_BOOL = 8, - "", // TYPE_STRING = 9, - "G", // TYPE_GROUP = 10, - "M", // TYPE_MESSAGE = 11, - "B", // TYPE_BYTES = 12, - "V32", // TYPE_UINT32 = 13, - "", // TYPE_ENUM = 14, - "F32", // TYPE_SFIXED32 = 15, - "F64", // TYPE_SFIXED64 = 16, - "Z32", // TYPE_SINT32 = 17, - "Z64", // TYPE_SINT64 = 18, - }; - name.append(kPrefix[field->type()]); - - if (field->type() == field->TYPE_ENUM) { - // Enums are handled as: - // - V32 for open enums - // - Er (and Er0/Er1) for sequential enums - // - Ev for the rest - if (cpp::HasPreservingUnknownEnumSemantics(field)) { - name.append("V32"); - } else { - int16_t start; - uint16_t size; - if (GetEnumValidationRange(field->enum_type(), start, size)) { - name.append("Er"); - int max_value = start + size - 1; - if (max_value <= 127 && (start == 0 || start == 1)) { - name.append(1, '0' + start); - aux_idx = max_value; - } - } else { - name.append("Ev"); - } - } - } - if (field->type() == field->TYPE_STRING) { - switch (internal::cpp::GetUtf8CheckMode(field, options.is_lite)) { - case internal::cpp::Utf8CheckMode::kStrict: - name.append("U"); - break; - case internal::cpp::Utf8CheckMode::kVerify: - name.append("S"); - break; - case internal::cpp::Utf8CheckMode::kNone: - name.append("B"); - break; - } - } - if (field->type() == field->TYPE_STRING || - field->type() == field->TYPE_BYTES) { - if (field->options().ctype() == FieldOptions::CORD) { - name.append("c"); - } else if (options.is_string_inlined) { - name.append("i"); + info.aux_idx = static_cast(entry.aux_idx); + if (field->type() == FieldDescriptor::TYPE_BYTES || + field->type() == FieldDescriptor::TYPE_STRING) { + if (options.is_string_inlined) { ABSL_CHECK(!field->is_repeated()); - aux_idx = static_cast(entry.inlined_string_idx); + info.aux_idx = static_cast(entry.inlined_string_idx); } } - if (field->type() == field->TYPE_MESSAGE || - field->type() == field->TYPE_GROUP) { - name.append(options.use_direct_tcparser_table ? "t" : "d"); - } - // The field implementation functions are prefixed by cardinality: - // `S` for optional or implicit fields. - // `R` for non-packed repeated. - // `P` for packed repeated. - name.append(field->is_packed() ? "P" - : field->is_repeated() ? "R" - : field->real_containing_oneof() ? "O" - : "S"); + TcParseFunction picked = TcParseFunction::kNone; + switch (field->type()) { + case FieldDescriptor::TYPE_BOOL: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastV8); + break; + case FieldDescriptor::TYPE_INT32: + case FieldDescriptor::TYPE_UINT32: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastV32); + break; + case FieldDescriptor::TYPE_SINT32: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastZ32); + break; + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_UINT64: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastV64); + break; + case FieldDescriptor::TYPE_SINT64: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastZ64); + break; + case FieldDescriptor::TYPE_FLOAT: + case FieldDescriptor::TYPE_FIXED32: + case FieldDescriptor::TYPE_SFIXED32: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastF32); + break; + case FieldDescriptor::TYPE_DOUBLE: + case FieldDescriptor::TYPE_FIXED64: + case FieldDescriptor::TYPE_SFIXED64: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastF64); + break; + case FieldDescriptor::TYPE_ENUM: + if (cpp::HasPreservingUnknownEnumSemantics(field)) { + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastV32); + } else { + switch (GetEnumRangeInfo(field, info.aux_idx)) { + case EnumRangeInfo::kNone: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastEv); + break; + case EnumRangeInfo::kContiguous: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastEr); + break; + case EnumRangeInfo::kContiguous0: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastEr0); + break; + case EnumRangeInfo::kContiguous1: + picked = PROTOBUF_PICK_PACKABLE_FUNCTION(kFastEr1); + break; + } + } + break; + case FieldDescriptor::TYPE_BYTES: + picked = PROTOBUF_PICK_STRING_FUNCTION(kFastB); + break; + case FieldDescriptor::TYPE_STRING: + switch (internal::cpp::GetUtf8CheckMode(field, options.is_lite)) { + case internal::cpp::Utf8CheckMode::kStrict: + picked = PROTOBUF_PICK_STRING_FUNCTION(kFastU); + break; + case internal::cpp::Utf8CheckMode::kVerify: + picked = PROTOBUF_PICK_STRING_FUNCTION(kFastS); + break; + case internal::cpp::Utf8CheckMode::kNone: + picked = PROTOBUF_PICK_STRING_FUNCTION(kFastB); + break; + } + break; + case FieldDescriptor::TYPE_MESSAGE: + picked = PROTOBUF_PICK_MESSAGE_FUNCTION(kFastM); + break; + case FieldDescriptor::TYPE_GROUP: + picked = PROTOBUF_PICK_MESSAGE_FUNCTION(kFastG); + break; + } - // Append the tag length. Fast parsing only handles 1- or 2-byte tags. - name.append(field->number() < 16 ? "1" : "2"); + ABSL_CHECK(picked != TcParseFunction::kNone); + static constexpr absl::string_view ns = "::_pbi::TcParser::"; + info.func_name = absl::StrCat(ns, ParseFunctionValue(picked)); - info.func_name = std::move(name); - info.aux_idx = aux_idx; +#undef PROTOBUF_PICK_FUNCTION +#undef PROTOBUF_PICK_SINGLE_FUNCTION +#undef PROTOBUF_PICK_REPEATABLE_FUNCTION +#undef PROTOBUF_PICK_PACKABLE_FUNCTION +#undef PROTOBUF_PICK_STRING_FUNCTION +#undef PROTOBUF_PICK_MESSAGE_FUNCTION } bool IsFieldEligibleForFastParsing( diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 13796053f916..35e5ec037c0a 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -267,6 +267,102 @@ inline void AlignFail(std::integral_constant, std::uintptr_t address) {} #endif +#define PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(fn) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##S1) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##S2) + +#define PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(fn) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(fn) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##R1) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##R2) + +#define PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(fn) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(fn) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##P1) \ + PROTOBUF_TC_PARSE_FUNCTION_X(fn##P2) + +// TcParseFunction defines the set of table driven, tail call optimized parse +// functions. This list currently does not include all types such as maps. +// +// This table identifies the logical set of functions, it does not imply that +// functions of the same name do exist, and some entries may point to thunks or +// generic implementations accepting multiple types of input. +// +// The names are encoded as follows: +// kFast[][cardinality][tag_width] +// +// type: +// V8 - bool +// V32 - int32/uint32 varint +// Z32 - int32/uint32 varint with zigzag encoding +// V64 - int64/uint64 varint +// Z64 - int64/uint64 varint with zigzag encoding +// F32 - int32/uint32/float fixed width value +// F64 - int64/uint64/double fixed width value +// E - enum +// B - string (bytes)* +// S - utf8 string, verified in debug mode only* +// U - utf8 string, strictly verified* +// Gd - group +// Gt - group width table driven parse tables +// Md - message +// Mt - message width table driven parse tables +// +// * string types can have a `c` or `i` suffix, indicating the +// underlying storage type to be cord or inlined respectively. +// +// validation: +// For enums: +// v - verify +// r - verify; enum values are a contiguous range +// r0 - verify; enum values are a small contiguous range starting at 0 +// r1 - verify; enum values are a small contiguous range starting at 1 +// For strings: +// u - validate utf8 encoding +// v - validate utf8 encoding for debug only +// +// cardinality: +// S - singular / optional +// R - repeated +// P - packed +// +// tag_width: +// 1: single byte encoded tag +// 2: two byte encoded tag +// +// Examples: +// FastV8S1, FastEr1P2, FastBcS1 +// +#define PROTOBUF_TC_PARSE_FUNCTION_LIST \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV8) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV32) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV64) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastZ32) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastZ64) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastF32) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastF64) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEv) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr0) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr1) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastB) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastS) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastU) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBi) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSi) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUi) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) + +#define PROTOBUF_TC_PARSE_FUNCTION_X(value) k##value, +enum class TcParseFunction { kNone, PROTOBUF_TC_PARSE_FUNCTION_LIST }; +#undef PROTOBUF_TC_PARSE_FUNCTION_X + // TcParser implements most of the parsing logic for tailcall tables. class PROTOBUF_EXPORT TcParser final { public: From 08c555755adbc50555fac8b726476f31e867627a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 23 Feb 2023 13:29:26 -0800 Subject: [PATCH 087/463] [ObjC] Ensure objects aren't leaked on extension parsing failures. For PODs, parse the data before making the NSNumber used for extension fields. For messages/groups, create the message first, add it to the object graph, and balance the retain count before doing the parsing. This ensure if an error is hit (and a throw happens), the object(s) won't be leaked. Parsing will always mutates the graph, so yes this includes more mutations in failure cases, but other fields could always be modified before the bad data is encountered. But even then, that edge case *only* apples to api users that are explicitly *merge* (-mergeFrom...), the majority of the calls are to +parseFromData:error:, so the entire graph is released on failure. PiperOrigin-RevId: 511871398 --- objectivec/GPBMessage.m | 202 +++++++++++++++++++++++++--------------- 1 file changed, 127 insertions(+), 75 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index a284666808d7..2e46ebea77b3 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -649,52 +649,93 @@ static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { #endif // !defined(__clang_analyzer__) -static id NewSingleValueFromInputStream(GPBExtensionDescriptor *extension, - GPBMessage *messageToGetExtension, - GPBCodedInputStream *input, - id extensionRegistry, - GPBMessage *existingValue) - __attribute__((ns_returns_retained)); - -// Note that this returns a retained value intentionally. -static id NewSingleValueFromInputStream(GPBExtensionDescriptor *extension, - GPBMessage *messageToGetExtension, - GPBCodedInputStream *input, - id extensionRegistry, - GPBMessage *existingValue) { +static void DecodeSingleValueFromInputStream(GPBExtensionDescriptor *extension, + GPBMessage *messageToGetExtension, + GPBCodedInputStream *input, + id extensionRegistry, + BOOL isRepeated, GPBMessage *targetMessage) { GPBExtensionDescription *description = extension->description_; +#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) + if (GPBDataTypeIsMessage(description->dataType)) { + NSCAssert(targetMessage != nil, @"Internal error: must have a target message"); + } else { + NSCAssert(targetMessage == nil, @"Internal error: should not have a target message"); + } +#endif GPBCodedInputStreamState *state = &input->state_; + id nsValue; switch (description->dataType) { - case GPBDataTypeBool: - return [[NSNumber alloc] initWithBool:GPBCodedInputStreamReadBool(state)]; - case GPBDataTypeFixed32: - return [[NSNumber alloc] initWithUnsignedInt:GPBCodedInputStreamReadFixed32(state)]; - case GPBDataTypeSFixed32: - return [[NSNumber alloc] initWithInt:GPBCodedInputStreamReadSFixed32(state)]; - case GPBDataTypeFloat: - return [[NSNumber alloc] initWithFloat:GPBCodedInputStreamReadFloat(state)]; - case GPBDataTypeFixed64: - return [[NSNumber alloc] initWithUnsignedLongLong:GPBCodedInputStreamReadFixed64(state)]; - case GPBDataTypeSFixed64: - return [[NSNumber alloc] initWithLongLong:GPBCodedInputStreamReadSFixed64(state)]; - case GPBDataTypeDouble: - return [[NSNumber alloc] initWithDouble:GPBCodedInputStreamReadDouble(state)]; - case GPBDataTypeInt32: - return [[NSNumber alloc] initWithInt:GPBCodedInputStreamReadInt32(state)]; - case GPBDataTypeInt64: - return [[NSNumber alloc] initWithLongLong:GPBCodedInputStreamReadInt64(state)]; - case GPBDataTypeSInt32: - return [[NSNumber alloc] initWithInt:GPBCodedInputStreamReadSInt32(state)]; - case GPBDataTypeSInt64: - return [[NSNumber alloc] initWithLongLong:GPBCodedInputStreamReadSInt64(state)]; - case GPBDataTypeUInt32: - return [[NSNumber alloc] initWithUnsignedInt:GPBCodedInputStreamReadUInt32(state)]; - case GPBDataTypeUInt64: - return [[NSNumber alloc] initWithUnsignedLongLong:GPBCodedInputStreamReadUInt64(state)]; + case GPBDataTypeBool: { + BOOL value = GPBCodedInputStreamReadBool(state); + nsValue = [[NSNumber alloc] initWithBool:value]; + break; + } + case GPBDataTypeFixed32: { + uint32_t value = GPBCodedInputStreamReadFixed32(state); + nsValue = [[NSNumber alloc] initWithUnsignedInt:value]; + break; + } + case GPBDataTypeSFixed32: { + int32_t value = GPBCodedInputStreamReadSFixed32(state); + nsValue = [[NSNumber alloc] initWithInt:value]; + break; + } + case GPBDataTypeFloat: { + float value = GPBCodedInputStreamReadFloat(state); + nsValue = [[NSNumber alloc] initWithFloat:value]; + break; + } + case GPBDataTypeFixed64: { + uint64_t value = GPBCodedInputStreamReadFixed64(state); + nsValue = [[NSNumber alloc] initWithUnsignedLongLong:value]; + break; + } + case GPBDataTypeSFixed64: { + int64_t value = GPBCodedInputStreamReadSFixed64(state); + nsValue = [[NSNumber alloc] initWithLongLong:value]; + break; + } + case GPBDataTypeDouble: { + double value = GPBCodedInputStreamReadDouble(state); + nsValue = [[NSNumber alloc] initWithDouble:value]; + break; + } + case GPBDataTypeInt32: { + int32_t value = GPBCodedInputStreamReadInt32(state); + nsValue = [[NSNumber alloc] initWithInt:value]; + break; + } + case GPBDataTypeInt64: { + int64_t value = GPBCodedInputStreamReadInt64(state); + nsValue = [[NSNumber alloc] initWithLongLong:value]; + break; + } + case GPBDataTypeSInt32: { + int32_t value = GPBCodedInputStreamReadSInt32(state); + nsValue = [[NSNumber alloc] initWithInt:value]; + break; + } + case GPBDataTypeSInt64: { + int64_t value = GPBCodedInputStreamReadSInt64(state); + nsValue = [[NSNumber alloc] initWithLongLong:value]; + break; + } + case GPBDataTypeUInt32: { + uint32_t value = GPBCodedInputStreamReadUInt32(state); + nsValue = [[NSNumber alloc] initWithUnsignedInt:value]; + break; + } + case GPBDataTypeUInt64: { + uint64_t value = GPBCodedInputStreamReadUInt64(state); + nsValue = [[NSNumber alloc] initWithUnsignedLongLong:value]; + break; + } case GPBDataTypeBytes: - return GPBCodedInputStreamReadRetainedBytes(state); + nsValue = GPBCodedInputStreamReadRetainedBytes(state); + break; case GPBDataTypeString: - return GPBCodedInputStreamReadRetainedString(state); + nsValue = GPBCodedInputStreamReadRetainedString(state); + break; case GPBDataTypeEnum: { int32_t val = GPBCodedInputStreamReadEnum(&input->state_); GPBEnumDescriptor *enumDescriptor = extension.enumDescriptor; @@ -702,43 +743,44 @@ static id NewSingleValueFromInputStream(GPBExtensionDescriptor *extension, // will be considers not closed, so casing to the enum type for a switch // could cause things to fall off the end of a switch. if (!enumDescriptor.isClosed || enumDescriptor.enumVerifier(val)) { - return [[NSNumber alloc] initWithInt:val]; + nsValue = [[NSNumber alloc] initWithInt:val]; } else { GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(messageToGetExtension); [unknownFields mergeVarintField:extension->description_->fieldNumber value:val]; - return nil; + nsValue = nil; } + break; } case GPBDataTypeGroup: case GPBDataTypeMessage: { - GPBMessage *message; - if (existingValue) { - message = [existingValue retain]; - } else { - GPBDescriptor *descriptor = [extension.msgClass descriptor]; - message = [[descriptor.messageClass alloc] init]; - } - if (description->dataType == GPBDataTypeGroup) { [input readGroup:description->fieldNumber - message:message + message:targetMessage extensionRegistry:extensionRegistry]; } else { // description->dataType == GPBDataTypeMessage if (GPBExtensionIsWireFormat(description)) { // For MessageSet fields the message length will have already been // read. - [message mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; + [targetMessage mergeFromCodedInputStream:input extensionRegistry:extensionRegistry]; } else { - [input readMessage:message extensionRegistry:extensionRegistry]; + [input readMessage:targetMessage extensionRegistry:extensionRegistry]; } } + // Nothing to add below since the caller provided the message (and added it). + nsValue = nil; + break; + } + } // switch - return message; + if (nsValue) { + if (isRepeated) { + [messageToGetExtension addExtension:extension value:nsValue]; + } else { + [messageToGetExtension setExtension:extension value:nsValue]; } + [nsValue release]; } - - return nil; } static void ExtensionMergeFromInputStream(GPBExtensionDescriptor *extension, BOOL isPackedOnStream, @@ -748,33 +790,40 @@ static void ExtensionMergeFromInputStream(GPBExtensionDescriptor *extension, BOO GPBExtensionDescription *description = extension->description_; GPBCodedInputStreamState *state = &input->state_; if (isPackedOnStream) { +#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) NSCAssert(GPBExtensionIsRepeated(description), @"How was it packed if it isn't repeated?"); +#endif int32_t length = GPBCodedInputStreamReadInt32(state); size_t limit = GPBCodedInputStreamPushLimit(state, length); while (GPBCodedInputStreamBytesUntilLimit(state) > 0) { - id value = NewSingleValueFromInputStream(extension, message, input, extensionRegistry, nil); - if (value) { - [message addExtension:extension value:value]; - [value release]; - } + DecodeSingleValueFromInputStream(extension, message, input, extensionRegistry, + /*isRepeated=*/YES, nil); } GPBCodedInputStreamPopLimit(state, limit); } else { - id existingValue = nil; BOOL isRepeated = GPBExtensionIsRepeated(description); - if (!isRepeated && GPBDataTypeIsMessage(description->dataType)) { - existingValue = [message getExistingExtension:extension]; - } - id value = - NewSingleValueFromInputStream(extension, message, input, extensionRegistry, existingValue); - if (value) { + GPBMessage *targetMessage = nil; + if (GPBDataTypeIsMessage(description->dataType)) { + // For messages/groups create the targetMessage out here and add it to the objects graph in + // advance, that way if DecodeSingleValueFromInputStream() throw for a parsing issue, the + // object won't be leaked. if (isRepeated) { - [message addExtension:extension value:value]; + GPBDescriptor *descriptor = [extension.msgClass descriptor]; + targetMessage = [[descriptor.messageClass alloc] init]; + [message addExtension:extension value:targetMessage]; + [targetMessage release]; } else { - [message setExtension:extension value:value]; + targetMessage = [message getExistingExtension:extension]; + if (!targetMessage) { + GPBDescriptor *descriptor = [extension.msgClass descriptor]; + targetMessage = [[descriptor.messageClass alloc] init]; + [message setExtension:extension value:targetMessage]; + [targetMessage release]; + } } - [value release]; } + DecodeSingleValueFromInputStream(extension, message, input, extensionRegistry, isRepeated, + targetMessage); } } @@ -2159,9 +2208,12 @@ - (void)parseMessageSet:(GPBCodedInputStream *)input if (rawBytes != nil && typeId != 0) { if (extension != nil) { GPBCodedInputStream *newInput = [[GPBCodedInputStream alloc] initWithData:rawBytes]; - ExtensionMergeFromInputStream(extension, extension.packable, newInput, extensionRegistry, - self); - [newInput release]; + @try { + ExtensionMergeFromInputStream(extension, extension.packable, newInput, extensionRegistry, + self); + } @finally { + [newInput release]; + } } else { GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); // rawBytes was created via a NoCopy, so it can be reusing a From d82d8a48f6c50ae6c811dbd6b7383e36a691c6b3 Mon Sep 17 00:00:00 2001 From: Jean byroot Boussier Date: Thu, 23 Feb 2023 13:44:39 -0800 Subject: [PATCH 088/463] Ruby: Implement Write Barriers (#11793) Write barrier protected objects are allowed to be promoted to the old generation, which means they only get marked on major GC. The downside is that the `RB_BJ_WRITE` macro MUST be used to set references, otherwise the referenced object may be garbaged collected. But the `*Descriptor` classes and `Arena` have very few references and are only set in a few places, so it's relatively easy to implement. cc @peterzhu2118 Closes #11793 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11793 from casperisfine:descriptor-write-barrier 215e8fad4cc5a5c9d0501e9d6c3114ab7db9f167 PiperOrigin-RevId: 511875342 --- ruby/ext/google/protobuf_c/defs.c | 34 +++++++++++++++++---------- ruby/ext/google/protobuf_c/message.c | 8 ++++--- ruby/ext/google/protobuf_c/protobuf.c | 6 +++-- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index fb1f5dc73fe6..75555fcb236d 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -223,6 +223,8 @@ static void DescriptorPool_register(VALUE module) { typedef struct { const upb_MessageDef* msgdef; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE klass; VALUE descriptor_pool; } Descriptor; @@ -238,7 +240,7 @@ static void Descriptor_mark(void* _self) { static const rb_data_type_t Descriptor_type = { "Google::Protobuf::Descriptor", {Descriptor_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static Descriptor* ruby_to_Descriptor(VALUE val) { @@ -280,7 +282,7 @@ static VALUE Descriptor_initialize(VALUE _self, VALUE cookie, "Descriptor objects may not be created from Ruby."); } - self->descriptor_pool = descriptor_pool; + RB_OBJ_WRITE(_self, &self->descriptor_pool, descriptor_pool); self->msgdef = (const upb_MessageDef*)NUM2ULL(ptr); return Qnil; @@ -390,7 +392,7 @@ static VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name) { static VALUE Descriptor_msgclass(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); if (self->klass == Qnil) { - self->klass = build_class_from_descriptor(_self); + RB_OBJ_WRITE(_self, &self->klass, build_class_from_descriptor(_self)); } return self->klass; } @@ -417,6 +419,8 @@ static void Descriptor_register(VALUE module) { typedef struct { const upb_FileDef* filedef; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE descriptor_pool; // Owns the upb_FileDef. } FileDescriptor; @@ -430,7 +434,7 @@ static void FileDescriptor_mark(void* _self) { static const rb_data_type_t FileDescriptor_type = { "Google::Protobuf::FileDescriptor", {FileDescriptor_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static FileDescriptor* ruby_to_FileDescriptor(VALUE val) { @@ -463,7 +467,7 @@ static VALUE FileDescriptor_initialize(VALUE _self, VALUE cookie, "Descriptor objects may not be created from Ruby."); } - self->descriptor_pool = descriptor_pool; + RB_OBJ_WRITE(_self, &self->descriptor_pool, descriptor_pool); self->filedef = (const upb_FileDef*)NUM2ULL(ptr); return Qnil; @@ -519,6 +523,8 @@ static void FileDescriptor_register(VALUE module) { typedef struct { const upb_FieldDef* fielddef; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE descriptor_pool; // Owns the upb_FieldDef. } FieldDescriptor; @@ -532,7 +538,7 @@ static void FieldDescriptor_mark(void* _self) { static const rb_data_type_t FieldDescriptor_type = { "Google::Protobuf::FieldDescriptor", {FieldDescriptor_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static FieldDescriptor* ruby_to_FieldDescriptor(VALUE val) { @@ -570,7 +576,7 @@ static VALUE FieldDescriptor_initialize(VALUE _self, VALUE cookie, "Descriptor objects may not be created from Ruby."); } - self->descriptor_pool = descriptor_pool; + RB_OBJ_WRITE(_self, &self->descriptor_pool, descriptor_pool); self->fielddef = (const upb_FieldDef*)NUM2ULL(ptr); return Qnil; @@ -884,6 +890,8 @@ static void FieldDescriptor_register(VALUE module) { typedef struct { const upb_OneofDef* oneofdef; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE descriptor_pool; // Owns the upb_OneofDef. } OneofDescriptor; @@ -897,7 +905,7 @@ static void OneofDescriptor_mark(void* _self) { static const rb_data_type_t OneofDescriptor_type = { "Google::Protobuf::OneofDescriptor", {OneofDescriptor_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static OneofDescriptor* ruby_to_OneofDescriptor(VALUE val) { @@ -936,7 +944,7 @@ static VALUE OneofDescriptor_initialize(VALUE _self, VALUE cookie, "Descriptor objects may not be created from Ruby."); } - self->descriptor_pool = descriptor_pool; + RB_OBJ_WRITE(_self, &self->descriptor_pool, descriptor_pool); self->oneofdef = (const upb_OneofDef*)NUM2ULL(ptr); return Qnil; @@ -988,6 +996,8 @@ static void OneofDescriptor_register(VALUE module) { typedef struct { const upb_EnumDef* enumdef; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE module; // begins as nil VALUE descriptor_pool; // Owns the upb_EnumDef. } EnumDescriptor; @@ -1003,7 +1013,7 @@ static void EnumDescriptor_mark(void* _self) { static const rb_data_type_t EnumDescriptor_type = { "Google::Protobuf::EnumDescriptor", {EnumDescriptor_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static EnumDescriptor* ruby_to_EnumDescriptor(VALUE val) { @@ -1042,7 +1052,7 @@ static VALUE EnumDescriptor_initialize(VALUE _self, VALUE cookie, "Descriptor objects may not be created from Ruby."); } - self->descriptor_pool = descriptor_pool; + RB_OBJ_WRITE(_self, &self->descriptor_pool, descriptor_pool); self->enumdef = (const upb_EnumDef*)NUM2ULL(ptr); return Qnil; @@ -1138,7 +1148,7 @@ static VALUE EnumDescriptor_each(VALUE _self) { static VALUE EnumDescriptor_enummodule(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); if (self->module == Qnil) { - self->module = build_module_from_enumdesc(_self); + RB_OBJ_WRITE(_self, &self->module, build_module_from_enumdesc(_self)); } return self->module; } diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 99e29466ebd3..067a44661576 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -53,6 +53,8 @@ VALUE MessageOrEnum_GetDescriptor(VALUE klass) { // ----------------------------------------------------------------------------- typedef struct { + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE arena; const upb_Message* msg; // Can get as mutable when non-frozen. const upb_MessageDef* @@ -65,9 +67,9 @@ static void Message_mark(void* _self) { } static rb_data_type_t Message_type = { - "Message", + "Google::Protobuf::Message", {Message_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static Message* ruby_to_Message(VALUE msg_rb) { @@ -105,7 +107,7 @@ upb_Message* Message_GetMutable(VALUE msg_rb, const upb_MessageDef** m) { void Message_InitPtr(VALUE self_, upb_Message* msg, VALUE arena) { Message* self = ruby_to_Message(self_); self->msg = msg; - self->arena = arena; + RB_OBJ_WRITE(self_, &self->arena, arena); ObjectCache_Add(msg, self_); } diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index 3c765c564d23..bee8ac288f71 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -171,6 +171,8 @@ void StringBuilder_PrintMsgval(StringBuilder *b, upb_MessageValue val, typedef struct { upb_Arena *arena; + // IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE() + // macro to update VALUE references, as to trigger write barriers. VALUE pinned_objs; } Arena; @@ -190,7 +192,7 @@ static VALUE cArena; const rb_data_type_t Arena_type = { "Google::Protobuf::Internal::Arena", {Arena_mark, Arena_free, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; static void* ruby_upb_allocfunc(upb_alloc* alloc, void* ptr, size_t oldsize, size_t size) { @@ -233,7 +235,7 @@ void Arena_Pin(VALUE _arena, VALUE obj) { Arena *arena; TypedData_Get_Struct(_arena, Arena, &Arena_type, arena); if (arena->pinned_objs == Qnil) { - arena->pinned_objs = rb_ary_new(); + RB_OBJ_WRITE(_arena, &arena->pinned_objs, rb_ary_new()); } rb_ary_push(arena->pinned_objs, obj); } From bcb20bbdfa0cba15c869d413edaaeb8507526a2e Mon Sep 17 00:00:00 2001 From: Greg Miller Date: Thu, 23 Feb 2023 15:07:15 -0800 Subject: [PATCH 089/463] Update the min required CMake version to 3.10 This change also removes policy settings that are now already set to `NEW` by default. There are still two remaining policies that were introduced in CMake > 3.10, so I left those, and documented the CMake version they were introduced in so we know when we can later remove them (when our min version advances beyond that). The min CMake version supported by Protobuf is documented here: https://github.com/google/oss-policies-info/blob/main/foundational-cxx-support-matrix.md PiperOrigin-RevId: 511897379 --- CMakeLists.txt | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 20786bd3ff4f..c068aae43cfc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,29 +1,18 @@ # Minimum CMake required -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) if(protobuf_VERBOSE) message(STATUS "Protocol Buffers Configuring...") endif() -# CMake policies -cmake_policy(SET CMP0022 NEW) -# On MacOS use @rpath/ for target's install name prefix path -if (POLICY CMP0042) - cmake_policy(SET CMP0042 NEW) -endif () -# Clear VERSION variables when no VERSION is given to project() -if(POLICY CMP0048) - cmake_policy(SET CMP0048 NEW) -endif() # MSVC runtime library flags are selected by an abstraction. +# New in CMake 3.15: https://cmake.org/cmake/help/latest/policy/CMP0091.html if(POLICY CMP0091) cmake_policy(SET CMP0091 NEW) endif() -# Honor visibility properties for all target types. -if(POLICY CMP0063) - cmake_policy(SET CMP0063 NEW) -endif() -# option() honor variables + +# option() honors normal variables +# New in CMake 3.13: https://cmake.org/cmake/help/latest/policy/CMP0077.html if (POLICY CMP0077) cmake_policy(SET CMP0077 NEW) endif (POLICY CMP0077) From 63389c027f474954e8178e77ac624e8ef952626d Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 23 Feb 2023 16:17:07 -0800 Subject: [PATCH 090/463] Add Python support for retention attribute PiperOrigin-RevId: 511914565 --- .../protobuf/internal/generator_test.py | 77 +++++++++++++ .../protobuf/compiler/python/BUILD.bazel | 12 +- .../protobuf/compiler/python/generator.cc | 100 +++++++++-------- .../protobuf/compiler/python/generator.h | 11 +- src/google/protobuf/compiler/retention.cc | 103 ++++++++++++++---- src/google/protobuf/compiler/retention.h | 22 ++++ 6 files changed, 251 insertions(+), 74 deletions(-) diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 291b97d9328d..a4370acbfc2a 100644 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -49,6 +49,7 @@ from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_wire_format_pb2 from google.protobuf import unittest_pb2 +from google.protobuf import unittest_retention_pb2 from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_no_generic_services_pb2 @@ -152,6 +153,82 @@ def testMessageWithCustomOptions(self): # TODO(gps): We really should test for the presence of the enum_opt1 # extension and for its value to be set to -789. + # Options that are explicitly marked RETENTION_SOURCE should not be present + # in the descriptors in the binary. + def testOptionRetention(self): + # Direct options + options = unittest_retention_pb2.DESCRIPTOR.GetOptions() + self.assertTrue(options.HasExtension(unittest_retention_pb2.plain_option)) + self.assertTrue( + options.HasExtension(unittest_retention_pb2.runtime_retention_option) + ) + self.assertFalse( + options.HasExtension(unittest_retention_pb2.source_retention_option) + ) + + def check_options_message_is_stripped_correctly(options): + self.assertEqual(options.plain_field, 1) + self.assertEqual(options.runtime_retention_field, 2) + self.assertFalse(options.HasField('source_retention_field')) + self.assertEqual(options.source_retention_field, 0) + + # Verify that our test OptionsMessage is stripped correctly on all + # different entity types. + check_options_message_is_stripped_correctly( + options.Extensions[unittest_retention_pb2.file_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.GetOptions().Extensions[ + unittest_retention_pb2.message_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.NestedMessage.DESCRIPTOR.GetOptions().Extensions[ + unittest_retention_pb2.message_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELENUM.GetOptions().Extensions[ + unittest_retention_pb2.enum_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELMESSAGE_NESTEDENUM.GetOptions().Extensions[ + unittest_retention_pb2.enum_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELENUM.values[0] + .GetOptions() + .Extensions[unittest_retention_pb2.enum_entry_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.extensions_by_name['i'] + .GetOptions() + .Extensions[unittest_retention_pb2.field_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.fields[0] + .GetOptions() + .Extensions[unittest_retention_pb2.field_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.oneofs[0] + .GetOptions() + .Extensions[unittest_retention_pb2.oneof_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] + .GetOptions() + .Extensions[unittest_retention_pb2.service_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] + .methods[0] + .GetOptions() + .Extensions[unittest_retention_pb2.method_option] + ) + def testNestedTypes(self): self.assertEqual( set(unittest_pb2.TestAllTypes.DESCRIPTOR.nested_types), diff --git a/src/google/protobuf/compiler/python/BUILD.bazel b/src/google/protobuf/compiler/python/BUILD.bazel index de1e06c8f355..90d5d21a8501 100644 --- a/src/google/protobuf/compiler/python/BUILD.bazel +++ b/src/google/protobuf/compiler/python/BUILD.bazel @@ -28,6 +28,7 @@ cc_library( deps = [ "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:retention", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -61,9 +62,12 @@ pkg_files( filegroup( name = "test_srcs", - srcs = glob([ - "*_test.cc", - "*unittest.cc", - ], allow_empty = True), + srcs = glob( + [ + "*_test.cc", + "*unittest.cc", + ], + allow_empty = True, + ), visibility = ["//src/google/protobuf/compiler:__pkg__"], ) diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index 58ffe0726cff..8aae2f1e0e66 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -64,6 +64,7 @@ #include "absl/strings/substitute.h" #include "google/protobuf/compiler/python/helpers.h" #include "google/protobuf/compiler/python/pyi_generator.h" +#include "google/protobuf/compiler/retention.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -249,8 +250,7 @@ bool Generator::Generate(const FileDescriptor* file, std::string filename = GetFileName(file, ".py"); - FileDescriptorProto fdp; - file_->CopyTo(&fdp); + FileDescriptorProto fdp = StripSourceRetentionOptions(*file_); fdp.SerializeToString(&file_descriptor_serialized_); if (!opensource_runtime_ && GeneratingDescriptorProto()) { @@ -342,7 +342,7 @@ bool Generator::Generate(const FileDescriptor* file, FixAllDescriptorOptions(); // Set serialized_start and serialized_end. - SetSerializedPbInterval(); + SetSerializedPbInterval(fdp); printer_->Outdent(); if (HasGenericServices(file)) { @@ -442,7 +442,8 @@ void Generator::PrintFileDescriptor() const { m["name"] = file_->name(); m["package"] = file_->package(); m["syntax"] = StringifySyntax(file_->syntax()); - m["options"] = OptionsValue(file_->options().SerializeAsString()); + m["options"] = OptionsValue( + StripLocalSourceRetentionOptions(*file_).SerializeAsString()); m["serialized_descriptor"] = absl::CHexEscape(file_descriptor_serialized_); if (GeneratingDescriptorProto()) { printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); @@ -528,7 +529,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { " create_key=_descriptor._internal_create_key,\n" " values=[\n"; std::string options_string; - enum_descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(enum_descriptor) + .SerializeToString(&options_string); printer_->Print(m, enum_descriptor_template); printer_->Indent(); printer_->Indent(); @@ -681,7 +683,8 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { printer_->Outdent(); printer_->Print("],\n"); std::string options_string; - message_descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(message_descriptor) + .SerializeToString(&options_string); printer_->Print( "serialized_options=$options_value$,\n" "is_extendable=$extendable$,\n" @@ -708,7 +711,8 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { m["name"] = desc->name(); m["full_name"] = desc->full_name(); m["index"] = absl::StrCat(desc->index()); - options_string = OptionsValue(desc->options().SerializeAsString()); + options_string = OptionsValue( + StripLocalSourceRetentionOptions(*desc).SerializeAsString()); if (options_string == "None") { m["serialized_options"] = ""; } else { @@ -1050,7 +1054,8 @@ void Generator::PrintEnumValueDescriptor( // TODO(robinson): Fix up EnumValueDescriptor "type" fields. // More circular references. ::sigh:: std::string options_string; - descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(descriptor) + .SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = descriptor.name(); m["index"] = absl::StrCat(descriptor.index()); @@ -1078,7 +1083,7 @@ std::string Generator::OptionsValue( void Generator::PrintFieldDescriptor(const FieldDescriptor& field, bool is_extension) const { std::string options_string; - field.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(field).SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = field.name(); m["full_name"] = field.full_name(); @@ -1216,21 +1221,17 @@ std::string Generator::InternalPackage() const { : "google3.net.google.protobuf.python.internal"; } -// Prints standard constructor arguments serialized_start and serialized_end. +// Prints descriptor offsets _serialized_start and _serialized_end. // Args: -// descriptor: The cpp descriptor to have a serialized reference. -// proto: A proto +// descriptor_proto: The descriptor proto to have a serialized reference. // Example printer output: -// serialized_start=41, -// serialized_end=43, -// -template -void Generator::PrintSerializedPbInterval(const DescriptorT& descriptor, - DescriptorProtoT& proto, - absl::string_view name) const { - descriptor.CopyTo(&proto); +// _globals['_MYMESSAGE']._serialized_start=47 +// _globals['_MYMESSAGE']._serialized_end=76 +template +void Generator::PrintSerializedPbInterval( + const DescriptorProtoT& descriptor_proto, absl::string_view name) const { std::string sp; - proto.SerializeToString(&sp); + descriptor_proto.SerializeToString(&sp); int offset = file_descriptor_serialized_.find(sp); ABSL_CHECK_GE(offset, 0); @@ -1254,43 +1255,47 @@ void PrintDescriptorOptionsFixingCode(absl::string_view descriptor, } } // namespace -void Generator::SetSerializedPbInterval() const { +// Generates the start and end offsets for each entity in the serialized file +// descriptor. The file argument must exactly match what was serialized into +// file_descriptor_serialized_, and should already have had any +// source-retention options stripped out. This is important because we need an +// exact byte-for-byte match so that we can successfully find the correct +// offsets in the serialized descriptors. +void Generator::SetSerializedPbInterval(const FileDescriptorProto& file) const { // Top level enums. for (int i = 0; i < file_->enum_type_count(); ++i) { - EnumDescriptorProto proto; const EnumDescriptor& descriptor = *file_->enum_type(i); - PrintSerializedPbInterval(descriptor, proto, + PrintSerializedPbInterval(file.enum_type(i), ModuleLevelDescriptorName(descriptor)); } // Messages. for (int i = 0; i < file_->message_type_count(); ++i) { - SetMessagePbInterval(*file_->message_type(i)); + SetMessagePbInterval(file.message_type(i), *file_->message_type(i)); } // Services. for (int i = 0; i < file_->service_count(); ++i) { - ServiceDescriptorProto proto; const ServiceDescriptor& service = *file_->service(i); - PrintSerializedPbInterval(service, proto, + PrintSerializedPbInterval(file.service(i), ModuleLevelServiceDescriptorName(service)); } } -void Generator::SetMessagePbInterval(const Descriptor& descriptor) const { - DescriptorProto message_proto; - PrintSerializedPbInterval(descriptor, message_proto, +void Generator::SetMessagePbInterval(const DescriptorProto& message_proto, + const Descriptor& descriptor) const { + PrintSerializedPbInterval(message_proto, ModuleLevelDescriptorName(descriptor)); // Nested messages. for (int i = 0; i < descriptor.nested_type_count(); ++i) { - SetMessagePbInterval(*descriptor.nested_type(i)); + SetMessagePbInterval(message_proto.nested_type(i), + *descriptor.nested_type(i)); } for (int i = 0; i < descriptor.enum_type_count(); ++i) { - EnumDescriptorProto proto; const EnumDescriptor& enum_des = *descriptor.enum_type(i); - PrintSerializedPbInterval(enum_des, proto, + PrintSerializedPbInterval(message_proto.enum_type(i), ModuleLevelDescriptorName(enum_des)); } } @@ -1298,7 +1303,8 @@ void Generator::SetMessagePbInterval(const Descriptor& descriptor) const { // Prints expressions that set the options field of all descriptors. void Generator::FixAllDescriptorOptions() const { // Prints an expression that sets the file descriptor's options. - std::string file_options = OptionsValue(file_->options().SerializeAsString()); + std::string file_options = OptionsValue( + StripLocalSourceRetentionOptions(*file_).SerializeAsString()); if (file_options != "None") { PrintDescriptorOptionsFixingCode(kDescriptorKey, file_options, printer_); } else { @@ -1326,7 +1332,8 @@ void Generator::FixAllDescriptorOptions() const { } void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { - std::string oneof_options = OptionsValue(oneof.options().SerializeAsString()); + std::string oneof_options = + OptionsValue(StripLocalSourceRetentionOptions(oneof).SerializeAsString()); if (oneof_options != "None") { std::string oneof_name = absl::Substitute( "$0.$1['$2']", ModuleLevelDescriptorName(*oneof.containing_type()), @@ -1339,15 +1346,15 @@ void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { // value descriptors. void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const { std::string descriptor_name = ModuleLevelDescriptorName(enum_descriptor); - std::string enum_options = - OptionsValue(enum_descriptor.options().SerializeAsString()); + std::string enum_options = OptionsValue( + StripLocalSourceRetentionOptions(enum_descriptor).SerializeAsString()); if (enum_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, enum_options, printer_); } for (int i = 0; i < enum_descriptor.value_count(); ++i) { const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(i); - std::string value_options = - OptionsValue(value_descriptor.options().SerializeAsString()); + std::string value_options = OptionsValue( + StripLocalSourceRetentionOptions(value_descriptor).SerializeAsString()); if (value_options != "None") { PrintDescriptorOptionsFixingCode( absl::StrFormat("%s.values_by_name[\"%s\"]", descriptor_name.c_str(), @@ -1363,8 +1370,8 @@ void Generator::FixOptionsForService( const ServiceDescriptor& service_descriptor) const { std::string descriptor_name = ModuleLevelServiceDescriptorName(service_descriptor); - std::string service_options = - OptionsValue(service_descriptor.options().SerializeAsString()); + std::string service_options = OptionsValue( + StripLocalSourceRetentionOptions(service_descriptor).SerializeAsString()); if (service_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, service_options, printer_); @@ -1372,8 +1379,8 @@ void Generator::FixOptionsForService( for (int i = 0; i < service_descriptor.method_count(); ++i) { const MethodDescriptor* method = service_descriptor.method(i); - std::string method_options = - OptionsValue(method->options().SerializeAsString()); + std::string method_options = OptionsValue( + StripLocalSourceRetentionOptions(*method).SerializeAsString()); if (method_options != "None") { std::string method_name = absl::StrCat( descriptor_name, ".methods_by_name['", method->name(), "']"); @@ -1385,7 +1392,8 @@ void Generator::FixOptionsForService( // Prints expressions that set the options for field descriptors (including // extensions). void Generator::FixOptionsForField(const FieldDescriptor& field) const { - std::string field_options = OptionsValue(field.options().SerializeAsString()); + std::string field_options = + OptionsValue(StripLocalSourceRetentionOptions(field).SerializeAsString()); if (field_options != "None") { std::string field_name; if (field.is_extension()) { @@ -1430,8 +1438,8 @@ void Generator::FixOptionsForMessage(const Descriptor& descriptor) const { FixOptionsForField(field); } // Message option for this message. - std::string message_options = - OptionsValue(descriptor.options().SerializeAsString()); + std::string message_options = OptionsValue( + StripLocalSourceRetentionOptions(descriptor).SerializeAsString()); if (message_options != "None") { std::string descriptor_name = ModuleLevelDescriptorName(descriptor); PrintDescriptorOptionsFixingCode(descriptor_name, message_options, diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index c5182c42c049..062b8fec1681 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -41,6 +41,7 @@ #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/descriptor.pb.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -166,9 +167,8 @@ class PROTOC_EXPORT Generator : public CodeGenerator { std::string PublicPackage() const; std::string InternalPackage() const; - template - void PrintSerializedPbInterval(const DescriptorT& descriptor, - DescriptorProtoT& proto, + template + void PrintSerializedPbInterval(const DescriptorProtoT& descriptor_proto, absl::string_view name) const; void FixAllDescriptorOptions() const; @@ -178,8 +178,9 @@ class PROTOC_EXPORT Generator : public CodeGenerator { void FixOptionsForService(const ServiceDescriptor& descriptor) const; void FixOptionsForMessage(const Descriptor& descriptor) const; - void SetSerializedPbInterval() const; - void SetMessagePbInterval(const Descriptor& descriptor) const; + void SetSerializedPbInterval(const FileDescriptorProto& file) const; + void SetMessagePbInterval(const DescriptorProto& message_proto, + const Descriptor& descriptor) const; void CopyPublicDependenciesAliases(absl::string_view copy_from, const FileDescriptor* file) const; diff --git a/src/google/protobuf/compiler/retention.cc b/src/google/protobuf/compiler/retention.cc index 087af7260c1a..04937a40e644 100644 --- a/src/google/protobuf/compiler/retention.cc +++ b/src/google/protobuf/compiler/retention.cc @@ -42,6 +42,7 @@ namespace protobuf { namespace compiler { namespace { + // Recursively strips any options with source retention from the message. void StripMessage(Message& m) { const Reflection* reflection = m.GetReflection(); @@ -62,43 +63,107 @@ void StripMessage(Message& m) { } } } -} // namespace -FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { - FileDescriptorProto file_proto; - file.CopyTo(&file_proto); - - // We need to look up the descriptor in file.pool() so that we can get a +// Converts the descriptor to a dynamic message if necessary, and then strips +// out all source-retention options. +// +// The options message may have custom options set on it, and these would +// ordinarily appear as unknown fields since they are not linked into protoc. +// Using a dynamic message allows us to see these custom options. To convert +// back and forth between the generated type and the dynamic message, we have +// to serialize one and parse that into the other. +void ConvertToDynamicMessageAndStripOptions(Message& m, + const DescriptorPool& pool) { + // We need to look up the descriptor in the pool so that we can get a // descriptor which knows about any custom options that were used in the // .proto file. - const Descriptor* descriptor = - file.pool()->FindMessageTypeByName(FileDescriptorProto().GetTypeName()); + const Descriptor* descriptor = pool.FindMessageTypeByName(m.GetTypeName()); if (descriptor == nullptr) { - // If the pool does not contain the descriptor for FileDescriptorProto, - // then this proto file does not transitively depend on descriptor.proto, - // in which case we know there are no custom options to worry about. - StripMessage(file_proto); + // If the pool does not contain the descriptor, then this proto file does + // not transitively depend on descriptor.proto, in which case we know there + // are no custom options to worry about. + StripMessage(m); } else { - // The options message may have custom options set on it, and these would - // ordinarily appear as unknown fields since they are not linked into - // protoc. Using a dynamic message allows us to see these custom options. - // To convert back and forth between the generated type and the dynamic - // message, we have to serialize one and parse that into the other. DynamicMessageFactory factory; std::unique_ptr dynamic_message( factory.GetPrototype(descriptor)->New()); std::string serialized; - ABSL_CHECK(file_proto.SerializeToString(&serialized)); + ABSL_CHECK(m.SerializeToString(&serialized)); ABSL_CHECK(dynamic_message->ParseFromString(serialized)); StripMessage(*dynamic_message); ABSL_CHECK(dynamic_message->SerializeToString(&serialized)); - ABSL_CHECK(file_proto.ParseFromString(serialized)); + ABSL_CHECK(m.ParseFromString(serialized)); } +} + +// Returns a const reference to the descriptor pool associated with the given +// descriptor. +template +const google::protobuf::DescriptorPool& GetPool(const DescriptorType& descriptor) { + return *descriptor.file()->pool(); +} + +// Specialization for FileDescriptor. +const google::protobuf::DescriptorPool& GetPool(const FileDescriptor& descriptor) { + return *descriptor.pool(); +} + +// Returns the options associated with the given descriptor, with all +// source-retention options stripped out. +template +auto StripLocalOptions(const DescriptorType& descriptor) { + auto options = descriptor.options(); + ConvertToDynamicMessageAndStripOptions(options, GetPool(descriptor)); + return options; +} +} // namespace + +FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { + FileDescriptorProto file_proto; + file.CopyTo(&file_proto); + ConvertToDynamicMessageAndStripOptions(file_proto, *file.pool()); return file_proto; } +EnumOptions StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +EnumValueOptions StripLocalSourceRetentionOptions( + const EnumValueDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +FieldOptions StripLocalSourceRetentionOptions( + const FieldDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +FileOptions StripLocalSourceRetentionOptions(const FileDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +MessageOptions StripLocalSourceRetentionOptions(const Descriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +MethodOptions StripLocalSourceRetentionOptions( + const MethodDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +OneofOptions StripLocalSourceRetentionOptions( + const OneofDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +ServiceOptions StripLocalSourceRetentionOptions( + const ServiceDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + } // namespace compiler } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/compiler/retention.h b/src/google/protobuf/compiler/retention.h index b97ab1e73a9e..0579b067e7f8 100644 --- a/src/google/protobuf/compiler/retention.h +++ b/src/google/protobuf/compiler/retention.h @@ -46,6 +46,28 @@ namespace compiler { PROTOC_EXPORT FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file); +// The following functions take a descriptor and strip all source-retention +// options from just the local entity (e.g. message, enum, field). Most code +// generators should not need these functions, but they are sometimes useful if +// you need to strip the options on a single entity rather than handling the +// entire file at once. +PROTOC_EXPORT EnumOptions +StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor); +PROTOC_EXPORT EnumValueOptions +StripLocalSourceRetentionOptions(const EnumValueDescriptor& descriptor); +PROTOC_EXPORT FieldOptions +StripLocalSourceRetentionOptions(const FieldDescriptor& descriptor); +PROTOC_EXPORT FileOptions +StripLocalSourceRetentionOptions(const FileDescriptor& descriptor); +PROTOC_EXPORT MessageOptions +StripLocalSourceRetentionOptions(const Descriptor& descriptor); +PROTOC_EXPORT MethodOptions +StripLocalSourceRetentionOptions(const MethodDescriptor& descriptor); +PROTOC_EXPORT OneofOptions +StripLocalSourceRetentionOptions(const OneofDescriptor& descriptor); +PROTOC_EXPORT ServiceOptions +StripLocalSourceRetentionOptions(const ServiceDescriptor& descriptor); + } // namespace compiler } // namespace protobuf } // namespace google From 2ce56399e30db62e45869c6fd2d2bbacbb81a7ed Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 23 Feb 2023 16:30:04 -0800 Subject: [PATCH 091/463] Ensure VarintParseSlowArm{32,64} are exported with PROTOBUF_EXPORT This should fix issue #11996. PiperOrigin-RevId: 511917449 --- src/google/protobuf/parse_context.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 0ddaa2604bf0..0169fe7828a8 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -667,7 +667,9 @@ inline const char* VarintParseSlow(const char* p, uint32_t res, uint64_t* out) { } #ifdef __aarch64__ +PROTOBUF_EXPORT const char* VarintParseSlowArm64(const char* p, uint64_t* out, uint64_t first8); +PROTOBUF_EXPORT const char* VarintParseSlowArm32(const char* p, uint32_t* out, uint64_t first8); inline const char* VarintParseSlowArm(const char* p, uint32_t* out, From 3c71dd852623352957d5b94a167199074d0c75f5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 02:28:06 -0800 Subject: [PATCH 092/463] nit: remove extra whitespace in NullValue comment PiperOrigin-RevId: 512017807 --- src/google/protobuf/struct.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/struct.proto b/src/google/protobuf/struct.proto index c4ea6453eaec..1bf0c1ad9586 100644 --- a/src/google/protobuf/struct.proto +++ b/src/google/protobuf/struct.proto @@ -80,7 +80,7 @@ message Value { // `NullValue` is a singleton enumeration to represent the null value for the // `Value` type union. // -// The JSON representation for `NullValue` is JSON `null`. +// The JSON representation for `NullValue` is JSON `null`. enum NullValue { // Null value. NULL_VALUE = 0; From 1501afeb23c698263eb3969c98c114597c14bb3b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 10:39:33 +0000 Subject: [PATCH 093/463] Auto-generate files after cl/512017807 --- csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 358617 -> 358616 bytes .../WellKnownTypes/Struct.pb.cs | 2 +- objectivec/GPBStruct.pbobjc.h | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 5e361f9027f6f0391fd4de880a8a3b04f7be217c..66c1815ce162cd19c96898c34a8fa5c290db4547 100644 GIT binary patch delta 68 zcmcclQS`<~(GBxvGydB=fA$?F#+jRI<}K&pTfxZ1#L6JV63h_BC^dVs;p&v>Tm4vM Xn>Vj+-@KX;h?#(xdHd$oENW)~1c4m* delta 67 zcmccdQS|0V(GBxvGydN^fA$?F##x(d<}K&pTgk}9#L6JV63h_BC^cuY;p&vhAD78B XZ(7~HX*DAdGXXL4_D!o<)Xo9`|BM}k diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.pb.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.pb.cs index aa25686dee03..b8c99c051c07 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.pb.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.pb.cs @@ -55,7 +55,7 @@ static StructReflection() { /// `NullValue` is a singleton enumeration to represent the null value for the /// `Value` type union. /// - /// The JSON representation for `NullValue` is JSON `null`. + /// The JSON representation for `NullValue` is JSON `null`. /// public enum NullValue { ///

diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 9c29c404bfd8c9d10dfab406f45edbf445b493c7 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Fri, 24 Feb 2023 08:08:52 -0800 Subject: [PATCH 094/463] [ObjC] Small cleanups to package level headers - Reduce what headers the runtime import needs. - Remove stale function prototype. - Move one function to be full scoped to a single file. PiperOrigin-RevId: 512073294 --- objectivec/GPBMessage.m | 10 ++++++++-- objectivec/GPBMessage_PackagePrivate.h | 9 --------- objectivec/GPBProtocolBuffers_RuntimeSupport.h | 3 +-- objectivec/GPBStruct.pbobjc.h | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 2e46ebea77b3..a6bedeb54617 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -47,6 +47,12 @@ #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUtilities_PackagePrivate.h" +// Returns a new instance that was automatically created by |autocreator| for +// its field |field|. +static GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, + GPBFieldDescriptor *field) + __attribute__((ns_returns_retained)); + // Direct access is use for speed, to avoid even internally declaring things // read/write, etc. The warning is enabled in the project to ensure code calling // protos can turn on -Wdirect-ivar-access without issues. @@ -827,8 +833,8 @@ static void ExtensionMergeFromInputStream(GPBExtensionDescriptor *extension, BOO } } -GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, - GPBFieldDescriptor *field) { +static GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, + GPBFieldDescriptor *field) { GPBMessage *message = [[msgClass alloc] init]; message->autocreator_ = autocreator; message->autocreatorField_ = [field retain]; diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 1b3a99332df8..8dc82e48489f 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -86,15 +86,6 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; CF_EXTERN_C_BEGIN -// Call this before using the readOnlySemaphore_. This ensures it is created only once. -void GPBPrepareReadOnlySemaphore(GPBMessage *self); - -// Returns a new instance that was automatically created by |autocreator| for -// its field |field|. -GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, - GPBFieldDescriptor *field) - __attribute__((ns_returns_retained)); - // Returns whether |message| autocreated this message. This is NO if the message // was not autocreated by |message| or if it has been mutated since // autocreation. diff --git a/objectivec/GPBProtocolBuffers_RuntimeSupport.h b/objectivec/GPBProtocolBuffers_RuntimeSupport.h index 804a68a73bd8..297173d74f7a 100644 --- a/objectivec/GPBProtocolBuffers_RuntimeSupport.h +++ b/objectivec/GPBProtocolBuffers_RuntimeSupport.h @@ -36,7 +36,6 @@ // clang-format on #import "GPBDescriptor_PackagePrivate.h" -#import "GPBExtensionInternals.h" -#import "GPBMessage_PackagePrivate.h" +#import "GPBMessage.h" #import "GPBRootObject_PackagePrivate.h" #import "GPBUtilities_PackagePrivate.h" diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index 8f2f12c73bcc..b0cc299714d3 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 0ece9236f54075410c1ac7b933f3f6abb3ee5c4c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 16:20:12 +0000 Subject: [PATCH 095/463] Auto-generate files after cl/512073294 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 086367f2c87512a6a916a62d8109cc614503a16d Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 08:24:33 -0800 Subject: [PATCH 096/463] Internal changes PiperOrigin-RevId: 512076306 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index 8f2f12c73bcc..b0cc299714d3 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From d32332dd81bbb8aef5fa142653f733d886d5d973 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 16:35:52 +0000 Subject: [PATCH 097/463] Auto-generate files after cl/512076306 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 0e48352e6df58da8653c48e09dfe3d843cef07f8 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 09:14:11 -0800 Subject: [PATCH 098/463] Expand lifetime of repository cache. These are not growing fast at all, but are taking a long time to asymptotically fill out. PiperOrigin-RevId: 512086597 --- .github/workflows/clear_caches.yml | 4 ++-- objectivec/GPBStruct.pbobjc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clear_caches.yml b/.github/workflows/clear_caches.yml index 3ea213d21ec2..f7d7bcedf240 100644 --- a/.github/workflows/clear_caches.yml +++ b/.github/workflows/clear_caches.yml @@ -2,8 +2,8 @@ name: Clear expensive caches to prevent unbounded growth on: schedule: - # Run every 1st of the month at 10 AM UTC (2 AM PDT) - - cron: 0 10 1 * * + # Run every 4 months at 10 AM UTC (2 AM PDT) + - cron: 0 10 1 */4 * # manual workflow_dispatch: diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index 8f2f12c73bcc..b0cc299714d3 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 3942e5307203c10bac2b8ea12b2cfcb65fcf2492 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 24 Feb 2023 09:20:28 -0800 Subject: [PATCH 099/463] Automated rollback of commit 63389c027f474954e8178e77ac624e8ef952626d. PiperOrigin-RevId: 512087869 --- .../protobuf/internal/generator_test.py | 77 ------------- .../protobuf/compiler/python/BUILD.bazel | 12 +- .../protobuf/compiler/python/generator.cc | 100 ++++++++--------- .../protobuf/compiler/python/generator.h | 11 +- src/google/protobuf/compiler/retention.cc | 103 ++++-------------- src/google/protobuf/compiler/retention.h | 22 ---- 6 files changed, 74 insertions(+), 251 deletions(-) diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index a4370acbfc2a..291b97d9328d 100644 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -49,7 +49,6 @@ from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_wire_format_pb2 from google.protobuf import unittest_pb2 -from google.protobuf import unittest_retention_pb2 from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_no_generic_services_pb2 @@ -153,82 +152,6 @@ def testMessageWithCustomOptions(self): # TODO(gps): We really should test for the presence of the enum_opt1 # extension and for its value to be set to -789. - # Options that are explicitly marked RETENTION_SOURCE should not be present - # in the descriptors in the binary. - def testOptionRetention(self): - # Direct options - options = unittest_retention_pb2.DESCRIPTOR.GetOptions() - self.assertTrue(options.HasExtension(unittest_retention_pb2.plain_option)) - self.assertTrue( - options.HasExtension(unittest_retention_pb2.runtime_retention_option) - ) - self.assertFalse( - options.HasExtension(unittest_retention_pb2.source_retention_option) - ) - - def check_options_message_is_stripped_correctly(options): - self.assertEqual(options.plain_field, 1) - self.assertEqual(options.runtime_retention_field, 2) - self.assertFalse(options.HasField('source_retention_field')) - self.assertEqual(options.source_retention_field, 0) - - # Verify that our test OptionsMessage is stripped correctly on all - # different entity types. - check_options_message_is_stripped_correctly( - options.Extensions[unittest_retention_pb2.file_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.GetOptions().Extensions[ - unittest_retention_pb2.message_option - ] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.TopLevelMessage.NestedMessage.DESCRIPTOR.GetOptions().Extensions[ - unittest_retention_pb2.message_option - ] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2._TOPLEVELENUM.GetOptions().Extensions[ - unittest_retention_pb2.enum_option - ] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2._TOPLEVELMESSAGE_NESTEDENUM.GetOptions().Extensions[ - unittest_retention_pb2.enum_option - ] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2._TOPLEVELENUM.values[0] - .GetOptions() - .Extensions[unittest_retention_pb2.enum_entry_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.DESCRIPTOR.extensions_by_name['i'] - .GetOptions() - .Extensions[unittest_retention_pb2.field_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.fields[0] - .GetOptions() - .Extensions[unittest_retention_pb2.field_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.oneofs[0] - .GetOptions() - .Extensions[unittest_retention_pb2.oneof_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] - .GetOptions() - .Extensions[unittest_retention_pb2.service_option] - ) - check_options_message_is_stripped_correctly( - unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] - .methods[0] - .GetOptions() - .Extensions[unittest_retention_pb2.method_option] - ) - def testNestedTypes(self): self.assertEqual( set(unittest_pb2.TestAllTypes.DESCRIPTOR.nested_types), diff --git a/src/google/protobuf/compiler/python/BUILD.bazel b/src/google/protobuf/compiler/python/BUILD.bazel index 90d5d21a8501..de1e06c8f355 100644 --- a/src/google/protobuf/compiler/python/BUILD.bazel +++ b/src/google/protobuf/compiler/python/BUILD.bazel @@ -28,7 +28,6 @@ cc_library( deps = [ "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", - "//src/google/protobuf/compiler:retention", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -62,12 +61,9 @@ pkg_files( filegroup( name = "test_srcs", - srcs = glob( - [ - "*_test.cc", - "*unittest.cc", - ], - allow_empty = True, - ), + srcs = glob([ + "*_test.cc", + "*unittest.cc", + ], allow_empty = True), visibility = ["//src/google/protobuf/compiler:__pkg__"], ) diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index 8aae2f1e0e66..58ffe0726cff 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -64,7 +64,6 @@ #include "absl/strings/substitute.h" #include "google/protobuf/compiler/python/helpers.h" #include "google/protobuf/compiler/python/pyi_generator.h" -#include "google/protobuf/compiler/retention.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -250,7 +249,8 @@ bool Generator::Generate(const FileDescriptor* file, std::string filename = GetFileName(file, ".py"); - FileDescriptorProto fdp = StripSourceRetentionOptions(*file_); + FileDescriptorProto fdp; + file_->CopyTo(&fdp); fdp.SerializeToString(&file_descriptor_serialized_); if (!opensource_runtime_ && GeneratingDescriptorProto()) { @@ -342,7 +342,7 @@ bool Generator::Generate(const FileDescriptor* file, FixAllDescriptorOptions(); // Set serialized_start and serialized_end. - SetSerializedPbInterval(fdp); + SetSerializedPbInterval(); printer_->Outdent(); if (HasGenericServices(file)) { @@ -442,8 +442,7 @@ void Generator::PrintFileDescriptor() const { m["name"] = file_->name(); m["package"] = file_->package(); m["syntax"] = StringifySyntax(file_->syntax()); - m["options"] = OptionsValue( - StripLocalSourceRetentionOptions(*file_).SerializeAsString()); + m["options"] = OptionsValue(file_->options().SerializeAsString()); m["serialized_descriptor"] = absl::CHexEscape(file_descriptor_serialized_); if (GeneratingDescriptorProto()) { printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); @@ -529,8 +528,7 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { " create_key=_descriptor._internal_create_key,\n" " values=[\n"; std::string options_string; - StripLocalSourceRetentionOptions(enum_descriptor) - .SerializeToString(&options_string); + enum_descriptor.options().SerializeToString(&options_string); printer_->Print(m, enum_descriptor_template); printer_->Indent(); printer_->Indent(); @@ -683,8 +681,7 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { printer_->Outdent(); printer_->Print("],\n"); std::string options_string; - StripLocalSourceRetentionOptions(message_descriptor) - .SerializeToString(&options_string); + message_descriptor.options().SerializeToString(&options_string); printer_->Print( "serialized_options=$options_value$,\n" "is_extendable=$extendable$,\n" @@ -711,8 +708,7 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { m["name"] = desc->name(); m["full_name"] = desc->full_name(); m["index"] = absl::StrCat(desc->index()); - options_string = OptionsValue( - StripLocalSourceRetentionOptions(*desc).SerializeAsString()); + options_string = OptionsValue(desc->options().SerializeAsString()); if (options_string == "None") { m["serialized_options"] = ""; } else { @@ -1054,8 +1050,7 @@ void Generator::PrintEnumValueDescriptor( // TODO(robinson): Fix up EnumValueDescriptor "type" fields. // More circular references. ::sigh:: std::string options_string; - StripLocalSourceRetentionOptions(descriptor) - .SerializeToString(&options_string); + descriptor.options().SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = descriptor.name(); m["index"] = absl::StrCat(descriptor.index()); @@ -1083,7 +1078,7 @@ std::string Generator::OptionsValue( void Generator::PrintFieldDescriptor(const FieldDescriptor& field, bool is_extension) const { std::string options_string; - StripLocalSourceRetentionOptions(field).SerializeToString(&options_string); + field.options().SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = field.name(); m["full_name"] = field.full_name(); @@ -1221,17 +1216,21 @@ std::string Generator::InternalPackage() const { : "google3.net.google.protobuf.python.internal"; } -// Prints descriptor offsets _serialized_start and _serialized_end. +// Prints standard constructor arguments serialized_start and serialized_end. // Args: -// descriptor_proto: The descriptor proto to have a serialized reference. +// descriptor: The cpp descriptor to have a serialized reference. +// proto: A proto // Example printer output: -// _globals['_MYMESSAGE']._serialized_start=47 -// _globals['_MYMESSAGE']._serialized_end=76 -template -void Generator::PrintSerializedPbInterval( - const DescriptorProtoT& descriptor_proto, absl::string_view name) const { +// serialized_start=41, +// serialized_end=43, +// +template +void Generator::PrintSerializedPbInterval(const DescriptorT& descriptor, + DescriptorProtoT& proto, + absl::string_view name) const { + descriptor.CopyTo(&proto); std::string sp; - descriptor_proto.SerializeToString(&sp); + proto.SerializeToString(&sp); int offset = file_descriptor_serialized_.find(sp); ABSL_CHECK_GE(offset, 0); @@ -1255,47 +1254,43 @@ void PrintDescriptorOptionsFixingCode(absl::string_view descriptor, } } // namespace -// Generates the start and end offsets for each entity in the serialized file -// descriptor. The file argument must exactly match what was serialized into -// file_descriptor_serialized_, and should already have had any -// source-retention options stripped out. This is important because we need an -// exact byte-for-byte match so that we can successfully find the correct -// offsets in the serialized descriptors. -void Generator::SetSerializedPbInterval(const FileDescriptorProto& file) const { +void Generator::SetSerializedPbInterval() const { // Top level enums. for (int i = 0; i < file_->enum_type_count(); ++i) { + EnumDescriptorProto proto; const EnumDescriptor& descriptor = *file_->enum_type(i); - PrintSerializedPbInterval(file.enum_type(i), + PrintSerializedPbInterval(descriptor, proto, ModuleLevelDescriptorName(descriptor)); } // Messages. for (int i = 0; i < file_->message_type_count(); ++i) { - SetMessagePbInterval(file.message_type(i), *file_->message_type(i)); + SetMessagePbInterval(*file_->message_type(i)); } // Services. for (int i = 0; i < file_->service_count(); ++i) { + ServiceDescriptorProto proto; const ServiceDescriptor& service = *file_->service(i); - PrintSerializedPbInterval(file.service(i), + PrintSerializedPbInterval(service, proto, ModuleLevelServiceDescriptorName(service)); } } -void Generator::SetMessagePbInterval(const DescriptorProto& message_proto, - const Descriptor& descriptor) const { - PrintSerializedPbInterval(message_proto, +void Generator::SetMessagePbInterval(const Descriptor& descriptor) const { + DescriptorProto message_proto; + PrintSerializedPbInterval(descriptor, message_proto, ModuleLevelDescriptorName(descriptor)); // Nested messages. for (int i = 0; i < descriptor.nested_type_count(); ++i) { - SetMessagePbInterval(message_proto.nested_type(i), - *descriptor.nested_type(i)); + SetMessagePbInterval(*descriptor.nested_type(i)); } for (int i = 0; i < descriptor.enum_type_count(); ++i) { + EnumDescriptorProto proto; const EnumDescriptor& enum_des = *descriptor.enum_type(i); - PrintSerializedPbInterval(message_proto.enum_type(i), + PrintSerializedPbInterval(enum_des, proto, ModuleLevelDescriptorName(enum_des)); } } @@ -1303,8 +1298,7 @@ void Generator::SetMessagePbInterval(const DescriptorProto& message_proto, // Prints expressions that set the options field of all descriptors. void Generator::FixAllDescriptorOptions() const { // Prints an expression that sets the file descriptor's options. - std::string file_options = OptionsValue( - StripLocalSourceRetentionOptions(*file_).SerializeAsString()); + std::string file_options = OptionsValue(file_->options().SerializeAsString()); if (file_options != "None") { PrintDescriptorOptionsFixingCode(kDescriptorKey, file_options, printer_); } else { @@ -1332,8 +1326,7 @@ void Generator::FixAllDescriptorOptions() const { } void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { - std::string oneof_options = - OptionsValue(StripLocalSourceRetentionOptions(oneof).SerializeAsString()); + std::string oneof_options = OptionsValue(oneof.options().SerializeAsString()); if (oneof_options != "None") { std::string oneof_name = absl::Substitute( "$0.$1['$2']", ModuleLevelDescriptorName(*oneof.containing_type()), @@ -1346,15 +1339,15 @@ void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { // value descriptors. void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const { std::string descriptor_name = ModuleLevelDescriptorName(enum_descriptor); - std::string enum_options = OptionsValue( - StripLocalSourceRetentionOptions(enum_descriptor).SerializeAsString()); + std::string enum_options = + OptionsValue(enum_descriptor.options().SerializeAsString()); if (enum_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, enum_options, printer_); } for (int i = 0; i < enum_descriptor.value_count(); ++i) { const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(i); - std::string value_options = OptionsValue( - StripLocalSourceRetentionOptions(value_descriptor).SerializeAsString()); + std::string value_options = + OptionsValue(value_descriptor.options().SerializeAsString()); if (value_options != "None") { PrintDescriptorOptionsFixingCode( absl::StrFormat("%s.values_by_name[\"%s\"]", descriptor_name.c_str(), @@ -1370,8 +1363,8 @@ void Generator::FixOptionsForService( const ServiceDescriptor& service_descriptor) const { std::string descriptor_name = ModuleLevelServiceDescriptorName(service_descriptor); - std::string service_options = OptionsValue( - StripLocalSourceRetentionOptions(service_descriptor).SerializeAsString()); + std::string service_options = + OptionsValue(service_descriptor.options().SerializeAsString()); if (service_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, service_options, printer_); @@ -1379,8 +1372,8 @@ void Generator::FixOptionsForService( for (int i = 0; i < service_descriptor.method_count(); ++i) { const MethodDescriptor* method = service_descriptor.method(i); - std::string method_options = OptionsValue( - StripLocalSourceRetentionOptions(*method).SerializeAsString()); + std::string method_options = + OptionsValue(method->options().SerializeAsString()); if (method_options != "None") { std::string method_name = absl::StrCat( descriptor_name, ".methods_by_name['", method->name(), "']"); @@ -1392,8 +1385,7 @@ void Generator::FixOptionsForService( // Prints expressions that set the options for field descriptors (including // extensions). void Generator::FixOptionsForField(const FieldDescriptor& field) const { - std::string field_options = - OptionsValue(StripLocalSourceRetentionOptions(field).SerializeAsString()); + std::string field_options = OptionsValue(field.options().SerializeAsString()); if (field_options != "None") { std::string field_name; if (field.is_extension()) { @@ -1438,8 +1430,8 @@ void Generator::FixOptionsForMessage(const Descriptor& descriptor) const { FixOptionsForField(field); } // Message option for this message. - std::string message_options = OptionsValue( - StripLocalSourceRetentionOptions(descriptor).SerializeAsString()); + std::string message_options = + OptionsValue(descriptor.options().SerializeAsString()); if (message_options != "None") { std::string descriptor_name = ModuleLevelDescriptorName(descriptor); PrintDescriptorOptionsFixingCode(descriptor_name, message_options, diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index 062b8fec1681..c5182c42c049 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -41,7 +41,6 @@ #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "google/protobuf/compiler/code_generator.h" -#include "google/protobuf/descriptor.pb.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -167,8 +166,9 @@ class PROTOC_EXPORT Generator : public CodeGenerator { std::string PublicPackage() const; std::string InternalPackage() const; - template - void PrintSerializedPbInterval(const DescriptorProtoT& descriptor_proto, + template + void PrintSerializedPbInterval(const DescriptorT& descriptor, + DescriptorProtoT& proto, absl::string_view name) const; void FixAllDescriptorOptions() const; @@ -178,9 +178,8 @@ class PROTOC_EXPORT Generator : public CodeGenerator { void FixOptionsForService(const ServiceDescriptor& descriptor) const; void FixOptionsForMessage(const Descriptor& descriptor) const; - void SetSerializedPbInterval(const FileDescriptorProto& file) const; - void SetMessagePbInterval(const DescriptorProto& message_proto, - const Descriptor& descriptor) const; + void SetSerializedPbInterval() const; + void SetMessagePbInterval(const Descriptor& descriptor) const; void CopyPublicDependenciesAliases(absl::string_view copy_from, const FileDescriptor* file) const; diff --git a/src/google/protobuf/compiler/retention.cc b/src/google/protobuf/compiler/retention.cc index 04937a40e644..087af7260c1a 100644 --- a/src/google/protobuf/compiler/retention.cc +++ b/src/google/protobuf/compiler/retention.cc @@ -42,7 +42,6 @@ namespace protobuf { namespace compiler { namespace { - // Recursively strips any options with source retention from the message. void StripMessage(Message& m) { const Reflection* reflection = m.GetReflection(); @@ -63,107 +62,43 @@ void StripMessage(Message& m) { } } } +} // namespace -// Converts the descriptor to a dynamic message if necessary, and then strips -// out all source-retention options. -// -// The options message may have custom options set on it, and these would -// ordinarily appear as unknown fields since they are not linked into protoc. -// Using a dynamic message allows us to see these custom options. To convert -// back and forth between the generated type and the dynamic message, we have -// to serialize one and parse that into the other. -void ConvertToDynamicMessageAndStripOptions(Message& m, - const DescriptorPool& pool) { - // We need to look up the descriptor in the pool so that we can get a +FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { + FileDescriptorProto file_proto; + file.CopyTo(&file_proto); + + // We need to look up the descriptor in file.pool() so that we can get a // descriptor which knows about any custom options that were used in the // .proto file. - const Descriptor* descriptor = pool.FindMessageTypeByName(m.GetTypeName()); + const Descriptor* descriptor = + file.pool()->FindMessageTypeByName(FileDescriptorProto().GetTypeName()); if (descriptor == nullptr) { - // If the pool does not contain the descriptor, then this proto file does - // not transitively depend on descriptor.proto, in which case we know there - // are no custom options to worry about. - StripMessage(m); + // If the pool does not contain the descriptor for FileDescriptorProto, + // then this proto file does not transitively depend on descriptor.proto, + // in which case we know there are no custom options to worry about. + StripMessage(file_proto); } else { + // The options message may have custom options set on it, and these would + // ordinarily appear as unknown fields since they are not linked into + // protoc. Using a dynamic message allows us to see these custom options. + // To convert back and forth between the generated type and the dynamic + // message, we have to serialize one and parse that into the other. DynamicMessageFactory factory; std::unique_ptr dynamic_message( factory.GetPrototype(descriptor)->New()); std::string serialized; - ABSL_CHECK(m.SerializeToString(&serialized)); + ABSL_CHECK(file_proto.SerializeToString(&serialized)); ABSL_CHECK(dynamic_message->ParseFromString(serialized)); StripMessage(*dynamic_message); ABSL_CHECK(dynamic_message->SerializeToString(&serialized)); - ABSL_CHECK(m.ParseFromString(serialized)); + ABSL_CHECK(file_proto.ParseFromString(serialized)); } -} - -// Returns a const reference to the descriptor pool associated with the given -// descriptor. -template -const google::protobuf::DescriptorPool& GetPool(const DescriptorType& descriptor) { - return *descriptor.file()->pool(); -} - -// Specialization for FileDescriptor. -const google::protobuf::DescriptorPool& GetPool(const FileDescriptor& descriptor) { - return *descriptor.pool(); -} - -// Returns the options associated with the given descriptor, with all -// source-retention options stripped out. -template -auto StripLocalOptions(const DescriptorType& descriptor) { - auto options = descriptor.options(); - ConvertToDynamicMessageAndStripOptions(options, GetPool(descriptor)); - return options; -} -} // namespace - -FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { - FileDescriptorProto file_proto; - file.CopyTo(&file_proto); - ConvertToDynamicMessageAndStripOptions(file_proto, *file.pool()); return file_proto; } -EnumOptions StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -EnumValueOptions StripLocalSourceRetentionOptions( - const EnumValueDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -FieldOptions StripLocalSourceRetentionOptions( - const FieldDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -FileOptions StripLocalSourceRetentionOptions(const FileDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -MessageOptions StripLocalSourceRetentionOptions(const Descriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -MethodOptions StripLocalSourceRetentionOptions( - const MethodDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -OneofOptions StripLocalSourceRetentionOptions( - const OneofDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - -ServiceOptions StripLocalSourceRetentionOptions( - const ServiceDescriptor& descriptor) { - return StripLocalOptions(descriptor); -} - } // namespace compiler } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/compiler/retention.h b/src/google/protobuf/compiler/retention.h index 0579b067e7f8..b97ab1e73a9e 100644 --- a/src/google/protobuf/compiler/retention.h +++ b/src/google/protobuf/compiler/retention.h @@ -46,28 +46,6 @@ namespace compiler { PROTOC_EXPORT FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file); -// The following functions take a descriptor and strip all source-retention -// options from just the local entity (e.g. message, enum, field). Most code -// generators should not need these functions, but they are sometimes useful if -// you need to strip the options on a single entity rather than handling the -// entire file at once. -PROTOC_EXPORT EnumOptions -StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor); -PROTOC_EXPORT EnumValueOptions -StripLocalSourceRetentionOptions(const EnumValueDescriptor& descriptor); -PROTOC_EXPORT FieldOptions -StripLocalSourceRetentionOptions(const FieldDescriptor& descriptor); -PROTOC_EXPORT FileOptions -StripLocalSourceRetentionOptions(const FileDescriptor& descriptor); -PROTOC_EXPORT MessageOptions -StripLocalSourceRetentionOptions(const Descriptor& descriptor); -PROTOC_EXPORT MethodOptions -StripLocalSourceRetentionOptions(const MethodDescriptor& descriptor); -PROTOC_EXPORT OneofOptions -StripLocalSourceRetentionOptions(const OneofDescriptor& descriptor); -PROTOC_EXPORT ServiceOptions -StripLocalSourceRetentionOptions(const ServiceDescriptor& descriptor); - } // namespace compiler } // namespace protobuf } // namespace google From 094511c858e75c5881a0c6d468efb28cf6afa84c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 17:31:39 +0000 Subject: [PATCH 100/463] Auto-generate files after cl/512087869 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From d6dc0dc2609488bec08bd5266816c57af4f396bc Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 09:45:23 -0800 Subject: [PATCH 101/463] Internal changes PiperOrigin-RevId: 512093266 --- objectivec/GPBStruct.pbobjc.h | 2 +- src/google/protobuf/compiler/rust/METADATA | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 src/google/protobuf/compiler/rust/METADATA diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index 8f2f12c73bcc..b0cc299714d3 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** diff --git a/src/google/protobuf/compiler/rust/METADATA b/src/google/protobuf/compiler/rust/METADATA deleted file mode 100644 index 9b19d6cdbd8b..000000000000 --- a/src/google/protobuf/compiler/rust/METADATA +++ /dev/null @@ -1,5 +0,0 @@ -# go/google3metadata -# proto-file: devtools/metadata/metadata.proto -# proto-message: MetaData - -presubmit: { review_notify: "protobuf-rust+reviews" } From 6a5467ada755244e8566b82e993c434d3b275280 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 17:57:04 +0000 Subject: [PATCH 102/463] Auto-generate files after cl/512093266 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 4bffbfa04acd9d9de21fadc0b0fc7977b3a854df Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Fri, 24 Feb 2023 09:53:19 -0800 Subject: [PATCH 103/463] Internal change PiperOrigin-RevId: 512095262 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index 8f2f12c73bcc..b0cc299714d3 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From 9ef627938aa9dde6eab91dd83b4fac1ebe84d82e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 10:31:14 -0800 Subject: [PATCH 104/463] Internal change PiperOrigin-RevId: 512105133 --- objectivec/GPBStruct.pbobjc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index b0cc299714d3..8f2f12c73bcc 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * `NullValue` is a singleton enumeration to represent the null value for the * `Value` type union. * - * The JSON representation for `NullValue` is JSON `null`. + * The JSON representation for `NullValue` is JSON `null`. **/ typedef GPB_ENUM(GPBNullValue) { /** From ef8ebe6c19008801ae49ae37db4ec5963e952483 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 11:18:04 -0800 Subject: [PATCH 105/463] Migrate our shared github actions to a separate repository. This will make PRs from forked repositories significantly less painful, since they'll agree on which version of each action to use. OTOH, we'll have a separate repo that needs to be maintained, and changes to it will need to be coordinated and versioned carefully. This will likely need to be done less often though now that our infrastructure is stable. PiperOrigin-RevId: 512117705 --- .github/actions/bash/action.yml | 27 ---- .github/actions/bazel-docker/action.yml | 75 ---------- .github/actions/bazel/action.yml | 134 ------------------ .github/actions/ccache/action.yml | 63 -------- .../actions/cross-compile-protoc/action.yml | 38 ----- .github/actions/docker/action.yml | 50 ------- .../actions/internal/bazel-setup/action.yml | 75 ---------- .../internal/ccache-setup-windows/action.yml | 68 --------- .../actions/internal/docker-run/action.yml | 62 -------- .../actions/internal/gcloud-auth/action.yml | 42 ------ .../repository-cache-restore/action.yml | 41 ------ .../internal/repository-cache-save/action.yml | 19 --- .../actions/internal/setup-runner/action.yml | 17 --- .github/workflows/staleness_check.yml | 2 +- .github/workflows/test_cpp.yml | 22 +-- .github/workflows/test_csharp.yml | 6 +- .github/workflows/test_java.yml | 4 +- .github/workflows/test_objectivec.yml | 8 +- .github/workflows/test_php.yml | 14 +- .github/workflows/test_python.yml | 4 +- .github/workflows/test_ruby.yml | 10 +- 21 files changed, 35 insertions(+), 746 deletions(-) delete mode 100644 .github/actions/bash/action.yml delete mode 100644 .github/actions/bazel-docker/action.yml delete mode 100644 .github/actions/bazel/action.yml delete mode 100644 .github/actions/ccache/action.yml delete mode 100644 .github/actions/cross-compile-protoc/action.yml delete mode 100644 .github/actions/docker/action.yml delete mode 100644 .github/actions/internal/bazel-setup/action.yml delete mode 100644 .github/actions/internal/ccache-setup-windows/action.yml delete mode 100644 .github/actions/internal/docker-run/action.yml delete mode 100644 .github/actions/internal/gcloud-auth/action.yml delete mode 100644 .github/actions/internal/repository-cache-restore/action.yml delete mode 100644 .github/actions/internal/repository-cache-save/action.yml delete mode 100644 .github/actions/internal/setup-runner/action.yml diff --git a/.github/actions/bash/action.yml b/.github/actions/bash/action.yml deleted file mode 100644 index d105a44f10bc..000000000000 --- a/.github/actions/bash/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: 'Non-Bazel Bash Run' -description: 'Run a bash script for Protobuf CI testing with a non-Bazel build system' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - command: - required: true - description: A command to run in the docker image - -runs: - using: 'composite' - steps: - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Update stale files using Bazel - uses: ./.github/actions/bazel - with: - credentials: ${{ inputs.credentials }} - bazel-cache: regenerate-stale-files - bash: ./regenerate_stale_files.sh $BAZEL_FLAGS - - - name: Run - shell: bash - run: ${{ inputs.command }} diff --git a/.github/actions/bazel-docker/action.yml b/.github/actions/bazel-docker/action.yml deleted file mode 100644 index af1104e23df8..000000000000 --- a/.github/actions/bazel-docker/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: 'Docker Bazel Run' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - image: - required: false - default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - description: "The docker image to use" - type: string - bazel-cache: - required: true - description: > - A unique path for the Bazel cache. This will trigger the generation - of a BAZEL_CACHE environment variable inside the container that provides - the appropriate flags for any bazel command. - type: string - bazel: - required: false - description: "The Bazel command to run" - type: string - bash: - required: false - description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs." - type: string - -runs: - using: 'composite' - steps: - - name: Authenticate - id: auth - uses: ./.github/actions/internal/gcloud-auth - with: - credentials: ${{ inputs.credentials }} - - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Setup Bazel - id: bazel - uses: ./.github/actions/internal/bazel-setup - with: - credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) - bazel-cache: ${{ inputs.bazel-cache }} - - - name: Hook up repository Cache - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV - - - name: Validate inputs - if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} - shell: bash - run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 - - - name: Run Bash Docker - uses: ./.github/actions/internal/docker-run - if: ${{ inputs.bash }} - with: - image: ${{ inputs.image }} - run-flags: --entrypoint "/bin/bash" - command: -l -c "${{ inputs.bash }}" - - - name: Run Bazel Docker - uses: ./.github/actions/internal/docker-run - if: ${{ !inputs.bash }} - with: - image: ${{ inputs.image }} - command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }} - - - name: Save Bazel repository cache - # Only allow repository cache updates during post-submits. - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml deleted file mode 100644 index 5fd3e9a90171..000000000000 --- a/.github/actions/bazel/action.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: 'Docker Bazel Run' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - credentials: - required: true - description: The GCP credentials to use for reading the docker image - type: string - bazel-cache: - required: true - description: > - A unique path for the Bazel cache. This will trigger the generation - of a BAZEL_CACHE environment variable inside the container that provides - the appropriate flags for any bazel command. - type: string - version: - required: false - description: A pinned Bazel version to use - default: '5.1.1' - type: string - bazel: - required: false - description: The Bazel command to run - type: string - bash: - required: false - description: > - A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be - available to use for bazel runs. - type: string - -runs: - using: 'composite' - steps: - - name: Authenticate - id: auth - uses: ./.github/actions/internal/gcloud-auth - with: - credentials: ${{ inputs.credentials }} - - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Setup Bazel - id: bazel - uses: ./.github/actions/internal/bazel-setup - with: - credentials-file: ${{ steps.auth.outputs.credentials-file }} - bazel-cache: ${{ inputs.bazel-cache }} - - - name: Get Linux bazelisk path - if: runner.os == 'Linux' - shell: bash - run: echo "BAZELISK_PATH=~/.cache/bazelisk" >> $GITHUB_ENV - - - name: Get MacOS bazelisk path - if: runner.os == 'macOS' - shell: bash - run: echo "BAZELISK_PATH=~/Library/Caches/bazelisk" >> $GITHUB_ENV - - - name: Get Windows bazelisk path - if: runner.os == 'Windows' - shell: bash - run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV - - - name: Cache Bazelisk - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.BAZELISK_PATH }} - key: bazel-${{ runner.os }}-${{ inputs.version }} - - - name: Restore Bazelisk - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.BAZELISK_PATH }} - key: bazel-${{ runner.os }}-${{ inputs.version }} - - - name: Hook up repository Cache - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV - - - name: Validate inputs - if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} - shell: bash - run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 - - - name: Pin Bazel version - shell: bash - run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV - - - name: Output Bazel version - shell: bash - run: bazelisk version - - # Bazel has multiple Xcode calls with hardcoded timeouts. Many of these - # end up timing out on our github runners, causing flakes on every mac - # build that invoked Bazel. To work around this, we manually inoke these - # calls before running Bazel to make sure they end up in Xcode's cache for - # quicker runs later. All of these calls are obtained from xcrun calls in - # https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl. - # See https://github.com/bazelbuild/bazel/issues/17437 for more details. - # TODO(b/269503614) Remove this once Bazel provides an official solution. - - name: Warm up Xcode - if: ${{ runner.os == 'macOS' }} - shell: bash - run: | - mkdir -p mac_bazel_workaround - bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS - XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation" - SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3" - COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3" - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc - - - name: Run Bash - if: ${{ inputs.bash }} - run: ${{ inputs.bash }} - shell: bash - - - name: Run Bazel - if: ${{ !inputs.bash }} - run: >- - bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} - ${{ inputs.bazel }} $BAZEL_FLAGS - shell: bash - - - name: Save Bazel repository cache - # Only allow repository cache updates during post-submits. - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target'}} - uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/actions/ccache/action.yml b/.github/actions/ccache/action.yml deleted file mode 100644 index 2d705503649d..000000000000 --- a/.github/actions/ccache/action.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: 'CCache Setup' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - cache-prefix: - required: true - description: A unique prefix to prevent cache pollution - type: string - support-modules: - required: false - description: Whether or not we need to support modules. This can result in extra cache misses. - -runs: - using: 'composite' - steps: - - name: Configure ccache environment variables - shell: bash - run: | - echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV - echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV - echo "CCACHE_DIRECT=true" >> $GITHUB_ENV - echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV - - - name: Setup ccache on Windows - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/internal/ccache-setup-windows - - name: Setup ccache on Mac - if: ${{ runner.os == 'macOS' }} - shell: bash - run: brew install ccache - - - name: Setup fixed path ccache caching - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: | - .ccache/** - !.ccache/lock - !.ccache/tmp - # Always push to a cache key unique to this commit. - key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} - # Select a cache to restore from with the follow order of preference: - # 1) The exact same commit we're running over - # 2) The latest cache from the current ref branch - # 3) The latest push to the base ref of a pull request - restore-keys: | - ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} - ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }} - ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} - - - name: Enable module support - if: ${{ inputs.support-modules }} - shell: bash - run: | - echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV - echo "CCACHE_DEPEND=true" >> $GITHUB_ENV - - - name: Zero out ccache - if: ${{ runner.os != 'Linux' }} - shell: bash - run: ccache -z diff --git a/.github/actions/cross-compile-protoc/action.yml b/.github/actions/cross-compile-protoc/action.yml deleted file mode 100644 index 23660af705ce..000000000000 --- a/.github/actions/cross-compile-protoc/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: 'Cross-compile protoc' -description: 'Produces a cross-compiled protoc binary for a target architecture' -inputs: - credentials: - required: true - description: The GCP credentials to use for reading the docker image - type: string - architecture: - required: true - description: The target architecture to build for - type: string -outputs: - protoc: - description: "Cross-compiled protoc location. Also output to $PROTOC" - value: ${{ steps.output.outputs.protoc }} - -runs: - using: 'composite' - steps: - - name: Cross compile protoc for ${{ inputs.architecture }} - uses: ./.github/actions/bazel-docker - with: - credentials: ${{ inputs.credentials }} - bazel-cache: xcompile-protoc/${{ inputs.architecture }} - bash: | - bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS - cp bazel-bin/protoc . - - - name: Set protoc environment variable - shell: bash - run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV - - - name: Extract binary - id: output - shell: bash - run: | - mv protoc $PROTOC - echo "protoc=$PROTOC" >> $GITHUB_OUTPUT diff --git a/.github/actions/docker/action.yml b/.github/actions/docker/action.yml deleted file mode 100644 index 54de3f908ec7..000000000000 --- a/.github/actions/docker/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: 'Docker Non-Bazel Run' -description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - command: - required: true - description: A command to run in the docker image - image: - required: false - default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - description: "The docker image to use" - type: string - platform: - required: false - description: "The platform to use for the image" - type: string - skip-staleness-check: - required: false - description: "Skip staleness checks" - type: boolean - -runs: - using: 'composite' - steps: - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Update stale files using Bazel - if: ${{ !inputs.skip-staleness-check }} - uses: ./.github/actions/bazel-docker - with: - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - credentials: ${{ inputs.credentials }} - bazel-cache: regenerate-stale-files - bash: ./regenerate_stale_files.sh $BAZEL_FLAGS - - - name: Generate docker flags - if: inputs.platform - shell: bash - run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV - - - name: Run Docker - uses: ./.github/actions/internal/docker-run - with: - image: ${{ inputs.image }} - command: ${{ inputs.command }} - run-flags: ${{ env.DOCKER_RUN_FLAGS }} diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml deleted file mode 100644 index 0ef5d5a2266a..000000000000 --- a/.github/actions/internal/bazel-setup/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Setup Bazel -description: Setup a Bazel environment for Protobuf CI testing -inputs: - credentials-file: - required: true - description: The GCP credentials file to use for caching - type: string - bazel-cache: - required: true - description: A unique path for the Bazel cache. - type: string - -outputs: - bazel-flags: - description: Bazel flags that should be sent to all Bazel invocations - value: ${{ steps.output.outputs.bazel-flags }} - bazel-startup-flags: - description: Bazel startup flags that should be sent to all Bazel invocations - value: ${{ steps.output.outputs.bazel-startup-flags }} - -runs: - using: 'composite' - steps: - - name: Initialize BAZEL environment variable - shell: bash - run: echo "BAZEL=bazelisk" >> $GITHUB_ENV - - - name: Initialize Windows startup flags - if: runner.os == 'Windows' - shell: bash - run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV - - - name: Initialize Bazel flags - shell: bash - run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV - - - name: Initialize Windows-specific Bazel flags - if: runner.os == 'Windows' - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV - - - name: Initialize MacOS-specific Bazel flags - if: runner.os == 'macOS' - shell: bash - run: | - echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV - echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV - - - name: Configure Bazel caching - # Skip bazel cache for local act runs due to issue with credential files - # and nested docker images - if: ${{ inputs.bazel-cache && !github.event.act_local_test }} - shell: bash - run: >- - echo "BAZEL_FLAGS=$BAZEL_FLAGS - --google_credentials=${{ inputs.credentials-file }} - --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV - - - name: Configure Bazel cache writing - # External runs should never write to our caches. - if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV - - - name: Output Bazel flags - id: output - shell: bash - run: | - echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT - echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT - - - name: Restore Bazel repository cache - uses: ./.github/actions/internal/repository-cache-restore - with: - bazel-cache: ${{ inputs.bazel-cache }} diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml deleted file mode 100644 index 7b9f9925b35d..000000000000 --- a/.github/actions/internal/ccache-setup-windows/action.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: 'CCache Setup' -description: 'Setup ccache for us in Windows CI' -inputs: - ccache-version: - required: false - default: '4.7.4' - description: A pinned version of ccache - type: string - -runs: - using: 'composite' - steps: - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - - name: Setup ccache path - shell: bash - run: | - echo "CCACHE_EXE_PATH=$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_ENV - echo "$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_PATH - - - name: Add ccache to Powershell path - shell: pwsh - run: echo "%LOCALAPPDATA%\ccache-${{ inputs.ccache-version }}-windows-x86_64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Setup caching of ccache download - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - id: ccache-cache - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.CCACHE_EXE_PATH }} - key: ccache-exe-${{ inputs.ccache-version }} - - - name: Restore ccache download - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - id: ccache-restore - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.CCACHE_EXE_PATH }} - key: ccache-exe-${{ inputs.ccache-version }} - - - name: Download ccache - shell: bash - if: ${{ steps.ccache-cache.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'}} - run: | - cd $LOCALAPPDATA - curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip - unzip ccache.zip - rm ccache.zip - ccache --version - - - name: Configure ccache environment variables - shell: pwsh - run: | - Write-Host $Env:GITHUB_REF - $cllocation = (Get-Command cl.exe).Path - echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - - name: Configure Windows-specific ccache environment variables - shell: bash - # Windows caches are about 2x larger than other platforms. - run: | - echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV diff --git a/.github/actions/internal/docker-run/action.yml b/.github/actions/internal/docker-run/action.yml deleted file mode 100644 index 1cc14894d83b..000000000000 --- a/.github/actions/internal/docker-run/action.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: 'Run Docker' -description: 'Run a docker image for Protobuf CI testing' -inputs: - image: - required: true - description: "The docker image to use" - type: string - command: - required: true - description: "A raw docker command to run" - type: string - run-flags: - required: false - description: "Additional flags to pass to docker run" - type: string - - # WARNING: loading from cache appears to be slower than pull! - docker-cache: - required: false - description: "Enabled caching of pulled docker images." - -runs: - using: 'composite' - steps: - - name: Authenticate for GAR use - shell: bash - run: gcloud auth configure-docker -q us-docker.pkg.dev - - - name: Setup QEMU for possible emulation - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - - - name: Check docker cache - if: ${{ inputs.docker-cache }} - id: check-docker-cache - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ci/docker/ - key: ${{ inputs.image }} - - - name: Pull and store if cache miss - shell: bash - if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} - run: > - time docker pull -q ${{ inputs.image }} && - mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && - time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar - - - name: Use the cached image on cache hit - shell: bash - if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} - run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar - - - name: Pull fresh docker image - shell: bash - if: ${{ !inputs.docker-cache }} - run: time docker pull -q ${{ inputs.image }} - - - name: Run docker - shell: bash - run: > - time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace - ${{ inputs.image }} ${{ inputs.command }} diff --git a/.github/actions/internal/gcloud-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml deleted file mode 100644 index 189619ddba3c..000000000000 --- a/.github/actions/internal/gcloud-auth/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: 'Authenticate for GCP' -description: 'Authenticate a workflow for Protobuf CI testing' -inputs: - credentials: - required: true - description: "The GCP credentials to use for GCP" - type: string - -outputs: - credentials-file: - description: "Credentials file generated for GCP" - value: ${{ steps.output.outputs.credentials-file }} - -runs: - using: 'composite' - steps: - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 - with: - credentials_json: ${{ inputs.credentials }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 - - name: Use gcloud CLI - shell: bash - run: gcloud info - - - name: Store credentials path - shell: bash - run: echo "CREDENTIALS_FILE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV - - - name: Fix credentials path (Windows) - if: ${{ runner.os == 'Windows' }} - # Bash commands in windows don't like the backslash in the file path. - # Assume we're running in the root directory and grab the base name. - shell: bash - run: echo "CREDENTIALS_FILE="$(basename ${CREDENTIALS_FILE//\\//}) >> $GITHUB_ENV - - - name: Output credentials file - id: output - shell: bash - run: echo "credentials-file=${{ env.CREDENTIALS_FILE }}" >> $GITHUB_OUTPUT diff --git a/.github/actions/internal/repository-cache-restore/action.yml b/.github/actions/internal/repository-cache-restore/action.yml deleted file mode 100644 index a5b4a0a924d8..000000000000 --- a/.github/actions/internal/repository-cache-restore/action.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Restore Repository Cache -description: Restore the Bazel repository cache from our github action cache -inputs: - bazel-cache: - required: true - description: A unique path for the Bazel cache. - type: string - -# By design, these actions will restore the latest cache for this branch/os, -# and only save a new version if something has changed. Initially this will -# cause a lot of churn, since each test has a slightly different set of -# repositories to download. Over time though, since we don't upload no-op -# changes, this should converge to a stable set of 3 caches per branch. Every -# run will update the current cache with a new test's repositories, until there -# are no unique ones left. -# -# This saves asymptotic space, since each one of these can get up to ~500 MB -# and Github prunes the cache after 10 GB. -runs: - using: 'composite' - steps: - - name: Setup Bazel repository cache variables - shell: bash - run: | - REPOSITORY_CACHE_BASE=repository-cache-${{ github.base_ref || github.ref_name }}-${{ runner.os }} - echo "REPOSITORY_CACHE_BASE=$REPOSITORY_CACHE_BASE" >> $GITHUB_ENV - echo "REPOSITORY_CACHE_NAME=$REPOSITORY_CACHE_BASE-${{ inputs.bazel-cache}}-${{ github.sha }}" >> $GITHUB_ENV - echo "REPOSITORY_CACHE_PATH=.repository-cache" >> $GITHUB_ENV - - - name: Restore Bazel repository cache - id: restore-cache - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} - key: ${{ env.REPOSITORY_CACHE_NAME }} - restore-keys: ${{ env.REPOSITORY_CACHE_BASE }} - - - name: Initialize BAZEL environment variable - if: ${{ steps.restore-cache.cache-hit }} - shell: bash - run: echo "REPOSITORY_CACHE_HASH=${{ hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}" >> $GITHUB_ENV diff --git a/.github/actions/internal/repository-cache-save/action.yml b/.github/actions/internal/repository-cache-save/action.yml deleted file mode 100644 index 1324b2bf21f2..000000000000 --- a/.github/actions/internal/repository-cache-save/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Restore Repository Cache -description: Restore the Bazel repository cache from our github action cache - -# Note: this action will only work if repository-cache-restore has already -# been called. All bazel actions should specify the repository_cache parameter -# using REPOSITORY_CACHE_PATH. -# -# We intentionally upload to REPOSITORY_CACHE_BASE to prevent a flood of new -# caches on any change. Only 1 job per os in each test run will be allowed to -# update the cache because they're all trying to write to the same location. -runs: - using: 'composite' - steps: - - name: Save modified Bazel repository cache - if: ${{ env.REPOSITORY_CACHE_HASH != hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }} - uses: actions/cache/save@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} - key: ${{ env.REPOSITORY_CACHE_BASE }}-${{ github.sha }} diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml deleted file mode 100644 index 64afdc3c7c3e..000000000000 --- a/.github/actions/internal/setup-runner/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Setup CI Runner -# TODO(b/267357823) Consider moving this to it's own repository so we can -# include the call to actions/checkout. -description: Setup any platform-specific adjustments we need to make for CI -runs: - using: 'composite' - steps: - - name: Fix Windows line breaks - if: runner.os == 'Windows' - shell: bash - run: find . -type f -print0 | xargs -0 d2u 2>/dev/null - - - name: Install bazelrc files - shell: bash - run: | - cp ci/*.bazelrc . - cp -f ${{ runner.os }}.bazelrc .bazelrc diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 63c4cf39ce1f..9b9c4e3fbec4 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -24,7 +24,7 @@ jobs: ref: ${{ matrix.branch}} - name: Run all staleness tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: staleness_check/${{ matrix.branch}}_${{ matrix.os.value }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index f846d80a8dff..1a63692a59dc 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -44,7 +44,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -84,12 +84,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-${{ matrix.name }} - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -106,12 +106,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-install - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -135,12 +135,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-32-bit - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea platform: linux/386 @@ -173,7 +173,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: test ${{ matrix.bazel }} @@ -215,14 +215,14 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: ${{ matrix.name }} # Install phase. - name: Configure CMake for install if: matrix.install-flags - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }} @@ -244,7 +244,7 @@ jobs: run: cmake --build . --target clean && rm CMakeCache.txt - name: Configure CMake - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }} diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 022d4f1f6c12..d85867f3e9fb 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -18,7 +18,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -57,7 +57,7 @@ jobs: - name: Build protobuf C# tests under x86_64 docker image # Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory # (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run) - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -74,7 +74,7 @@ jobs: # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, # we just run map the user's home to a throwaway temporary directory - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 skip-staleness-check: true diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 1ad65537a1bf..bb6ea697e90a 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -35,7 +35,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -51,7 +51,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Build protoc - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: java_linux_linkage diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 4093b180b724..019bd4c95e34 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -36,13 +36,13 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }} support-modules: true - name: Run tests - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 env: CC: ${{ github.workspace }}/ci/clang_wrapper CXX: ${{ github.workspace }}/ci/clang_wrapper++ @@ -78,7 +78,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Pod lib lint - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: | @@ -107,7 +107,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: test ${{ matrix.config.flags }} ${{ matrix.bazel_targets }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 2d93d8f2b21a..4b9a6d02efa4 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -43,7 +43,7 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -78,13 +78,13 @@ jobs: - name: Cross compile protoc for i386 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: ${{ env.image }} skip-staleness-check: true @@ -109,13 +109,13 @@ jobs: - name: Cross compile protoc for aarch64 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310 platform: linux/arm64 @@ -157,7 +157,7 @@ jobs: run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1) - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_macos/${{ matrix.version }} @@ -171,7 +171,7 @@ jobs: popd - name: Run conformance tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_macos/${{ matrix.version }} diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 04fa54acc0a5..13e5575b24c2 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -37,7 +37,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -82,7 +82,7 @@ jobs: source venv/bin/activate - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 env: KOKORO_PYTHON_VERSION: ${{ matrix.version }} with: diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index f5e3663fd2b2..0f9392f526dd 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -32,7 +32,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -50,13 +50,13 @@ jobs: - name: Cross compile protoc for aarch64 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: arm64v8/ruby:2.7.3-buster skip-staleness-check: true @@ -94,7 +94,7 @@ jobs: run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1) - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_macos/${{ matrix.version }} @@ -122,7 +122,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} From 8ee8d3522b2ba35f487bcd78b23d522291602e18 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 11:58:19 -0800 Subject: [PATCH 106/463] Internal change PiperOrigin-RevId: 512128206 --- src/google/protobuf/extension_set.h | 6 +- src/google/protobuf/lazy_field_heavy.cc | 105 ++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 src/google/protobuf/lazy_field_heavy.cc diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 99b0e397f8fc..a80bf9db2bc8 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -575,9 +575,9 @@ class PROTOBUF_EXPORT ExtensionSet { virtual void MergeFromMessage(const MessageLite& msg, Arena* arena) = 0; virtual void Clear() = 0; - virtual const char* _InternalParse(const Message& prototype, Arena* arena, - LazyVerifyOption option, const char* ptr, - ParseContext* ctx) = 0; + virtual const char* _InternalParse(const MessageLite& prototype, + Arena* arena, LazyVerifyOption option, + const char* ptr, ParseContext* ctx) = 0; virtual uint8_t* WriteMessageToArray( const MessageLite* prototype, int number, uint8_t* target, io::EpsCopyOutputStream* stream) const = 0; diff --git a/src/google/protobuf/lazy_field_heavy.cc b/src/google/protobuf/lazy_field_heavy.cc new file mode 100644 index 000000000000..02d4d2ebfb4e --- /dev/null +++ b/src/google/protobuf/lazy_field_heavy.cc @@ -0,0 +1,105 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/lazy_field.h" +#include "google/protobuf/message.h" + +namespace google { +namespace protobuf { +namespace internal { +namespace { +class ByFactory { + public: + explicit ByFactory(const Descriptor* type, MessageFactory* factory) + : type_(type), factory_(factory) {} + + Message* New(Arena* arena) const { + return factory_->GetPrototype(type_)->New(arena); + } + + const Message& Default() const { return *factory_->GetPrototype(type_); } + + private: + const Descriptor* type_; + MessageFactory* factory_; +}; + +} // namespace + +const Message& LazyField::GetDynamic(const Descriptor* type, + MessageFactory* factory, + Arena* arena) const { + return DownCast(GetGeneric(ByFactory(type, factory), arena)); +} + +Message* LazyField::MutableDynamic(const Descriptor* type, + MessageFactory* factory, Arena* arena) { + return DownCast( + MutableGeneric(ByFactory(type, factory), arena, nullptr)); +} + +Message* LazyField::ReleaseDynamic(const Descriptor* type, + MessageFactory* factory, Arena* arena) { + return DownCast(ReleaseGeneric(ByFactory(type, factory), arena)); +} + +Message* LazyField::UnsafeArenaReleaseDynamic(const Descriptor* type, + MessageFactory* factory, + Arena* arena) { + return DownCast( + UnsafeArenaReleaseGeneric(ByFactory(type, factory), arena)); +} + +size_t LazyField::SpaceUsedExcludingSelfLong() const { + // absl::Cord::EstimatedMemoryUsage counts itself that should be excluded + // because sizeof(Cord) is already counted in self. + size_t total_size = unparsed_.EstimatedMemoryUsage() - sizeof(absl::Cord); + switch (GetLogicalState()) { + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: + case LogicalState::kParseError: { + const auto* message = raw_.load(std::memory_order_relaxed).message(); + total_size += DownCast(message)->SpaceUsedLong(); + } break; + case LogicalState::kClear: + case LogicalState::kParseRequired: + // We may have a `Message*` here, but we cannot safely access it + // because, a racing SharedInit could delete it out from under us. + // Other states in this structure are already passed kSharedInit and are + // thus safe. + break; // Nothing to add. + } + return total_size; +} + +} // namespace internal +} // namespace protobuf +} // namespace google From de4ce2e2de502e98c6ce64f3f7552f51703b1cd6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 12:46:15 -0800 Subject: [PATCH 107/463] Change the C++ parsers to be consistent on how they handle unknown enum values. They now consistently parse the varint and then push the int32 representation into the unknown field set. Previously the behavior was sometimes pushing the original varint, sometimes pushing a uint32_t truncated value, and sometimes an int32_t truncated value. PiperOrigin-RevId: 512140469 --- .../compiler/cpp/parse_function_generator.cc | 40 ++--- src/google/protobuf/descriptor.pb.cc | 18 +-- src/google/protobuf/extension_set_inl.h | 8 +- .../protobuf/generated_message_tctable_impl.h | 5 +- .../generated_message_tctable_lite.cc | 46 ++++-- src/google/protobuf/message_unittest.inc | 143 ++++++++++++++---- src/google/protobuf/parse_context.h | 4 +- src/google/protobuf/unittest.proto | 34 +++++ src/google/protobuf/wire_format.cc | 2 +- 9 files changed, 211 insertions(+), 89 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 791a7d5503bd..a296d411e71c 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -216,17 +216,6 @@ void ParseFunctionGenerator::GenerateTailcallParseFunction(Formatter& format) { "}\n\n"); } -static bool NeedsUnknownEnumSupport(const Descriptor* descriptor) { - for (int i = 0; i < descriptor->field_count(); ++i) { - auto* field = descriptor->field(i); - if (field->is_repeated() && field->cpp_type() == field->CPPTYPE_ENUM && - !internal::cpp::HasPreservingUnknownEnumSemantics(field)) { - return true; - } - } - return false; -} - void ParseFunctionGenerator::GenerateTailcallFallbackFunction( Formatter& format) { ABSL_CHECK(should_generate_tctable()); @@ -236,21 +225,18 @@ void ParseFunctionGenerator::GenerateTailcallFallbackFunction( format.Indent(); format("auto* typed_msg = static_cast<$classname$*>(msg);\n"); - // If we need a side channel, generate the check to jump to the generic - // handler to deal with the side channel data. - if (NeedsUnknownEnumSupport(descriptor_)) { - format( - "if (PROTOBUF_PREDICT_FALSE(\n" - " _pbi::TcParser::MustFallbackToGeneric(PROTOBUF_TC_PARAM_PASS))) " - "{\n" - " PROTOBUF_MUSTTAIL return " - "::_pbi::TcParser::GenericFallback$1$(PROTOBUF_TC_PARAM_PASS);\n" - "}\n", - GetOptimizeFor(descriptor_->file(), options_) == - FileOptions::LITE_RUNTIME - ? "Lite" - : ""); - } + // Generate the check to jump to the generic handler to deal with the side + // channel data. + format( + "if (PROTOBUF_PREDICT_FALSE(\n" + " _pbi::TcParser::MustFallbackToGeneric(PROTOBUF_TC_PARAM_PASS))) " + "{\n" + " PROTOBUF_MUSTTAIL return " + "::_pbi::TcParser::GenericFallback$1$(PROTOBUF_TC_PARAM_PASS);\n" + "}\n", + GetOptimizeFor(descriptor_->file(), options_) == FileOptions::LITE_RUNTIME + ? "Lite" + : ""); if (num_hasbits_ > 0) { // Sync hasbits @@ -1147,7 +1133,7 @@ void ParseFunctionGenerator::GenerateFieldBody( format.Set("enum_type", QualifiedClassName(field->enum_type(), options_)); format( - "$uint32$ val = ::$proto_ns$::internal::ReadVarint32(&ptr);\n" + "$int32$ val = ::$proto_ns$::internal::ReadVarint32(&ptr);\n" "CHK_(ptr);\n"); if (!internal::cpp::HasPreservingUnknownEnumSemantics(field)) { format( diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index ce8901ec0e62..88e9833a480e 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -4252,7 +4252,7 @@ const char* FieldDescriptorProto::_InternalParse(const char* ptr, ::_pbi::ParseC // optional .google.protobuf.FieldDescriptorProto.Label label = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 32)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto_Label_IsValid(static_cast(val)))) { _internal_set_label(static_cast<::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto_Label>(val)); @@ -4266,7 +4266,7 @@ const char* FieldDescriptorProto::_InternalParse(const char* ptr, ::_pbi::ParseC // optional .google.protobuf.FieldDescriptorProto.Type type = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 40)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto_Type_IsValid(static_cast(val)))) { _internal_set_type(static_cast<::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto_Type>(val)); @@ -7026,7 +7026,7 @@ const char* FileOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* c // optional .google.protobuf.FileOptions.OptimizeMode optimize_for = 9 [default = SPEED]; case 9: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 72)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FileOptions_OptimizeMode_IsValid(static_cast(val)))) { _internal_set_optimize_for(static_cast<::PROTOBUF_NAMESPACE_ID::FileOptions_OptimizeMode>(val)); @@ -8297,7 +8297,7 @@ const char* FieldOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* // optional .google.protobuf.FieldOptions.CType ctype = 1 [default = STRING]; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 8)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldOptions_CType_IsValid(static_cast(val)))) { _internal_set_ctype(static_cast<::PROTOBUF_NAMESPACE_ID::FieldOptions_CType>(val)); @@ -8341,7 +8341,7 @@ const char* FieldOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* // optional .google.protobuf.FieldOptions.JSType jstype = 6 [default = JS_NORMAL]; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 48)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldOptions_JSType_IsValid(static_cast(val)))) { _internal_set_jstype(static_cast<::PROTOBUF_NAMESPACE_ID::FieldOptions_JSType>(val)); @@ -8385,7 +8385,7 @@ const char* FieldOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* // optional .google.protobuf.FieldOptions.OptionRetention retention = 17; case 17: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 136)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldOptions_OptionRetention_IsValid(static_cast(val)))) { _internal_set_retention(static_cast<::PROTOBUF_NAMESPACE_ID::FieldOptions_OptionRetention>(val)); @@ -8399,7 +8399,7 @@ const char* FieldOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* // optional .google.protobuf.FieldOptions.OptionTargetType target = 18; case 18: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 144)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::FieldOptions_OptionTargetType_IsValid(static_cast(val)))) { _internal_set_target(static_cast<::PROTOBUF_NAMESPACE_ID::FieldOptions_OptionTargetType>(val)); @@ -9877,7 +9877,7 @@ const char* MethodOptions::_InternalParse(const char* ptr, ::_pbi::ParseContext* // optional .google.protobuf.MethodOptions.IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; case 34: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 16)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::MethodOptions_IdempotencyLevel_IsValid(static_cast(val)))) { _internal_set_idempotency_level(static_cast<::PROTOBUF_NAMESPACE_ID::MethodOptions_IdempotencyLevel>(val)); @@ -11581,7 +11581,7 @@ const char* GeneratedCodeInfo_Annotation::_InternalParse(const char* ptr, ::_pbi // optional .google.protobuf.GeneratedCodeInfo.Annotation.Semantic semantic = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 40)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); if (PROTOBUF_PREDICT_TRUE(::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo_Annotation_Semantic_IsValid(static_cast(val)))) { _internal_set_semantic(static_cast<::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo_Annotation_Semantic>(val)); diff --git a/src/google/protobuf/extension_set_inl.h b/src/google/protobuf/extension_set_inl.h index bd01e950440a..4554d950831d 100644 --- a/src/google/protobuf/extension_set_inl.h +++ b/src/google/protobuf/extension_set_inl.h @@ -141,14 +141,14 @@ const char* ExtensionSet::ParseFieldWithExtensionInfo( #undef HANDLE_FIXED_TYPE case WireFormatLite::TYPE_ENUM: { - uint64_t val; - ptr = VarintParse(ptr, &val); + uint64_t tmp; + ptr = VarintParse(ptr, &tmp); GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); - int value = val; + int value = tmp; if (!extension.enum_validity_check.func( extension.enum_validity_check.arg, value)) { - WriteVarint(number, val, metadata->mutable_unknown_fields()); + WriteVarint(number, value, metadata->mutable_unknown_fields()); } else if (extension.is_repeated) { AddEnum(number, WireFormatLite::TYPE_ENUM, extension.is_packed, value, extension.descriptor); diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 35e5ec037c0a..541b4750901a 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -655,6 +655,7 @@ class PROTOBUF_EXPORT TcParser final { static const char* Error(PROTOBUF_TC_PARAM_NO_DATA_DECL); static const char* FastUnknownEnumFallback(PROTOBUF_TC_PARAM_DECL); + static const char* MpUnknownEnumFallback(PROTOBUF_TC_PARAM_DECL); class ScopedArenaSwap; @@ -763,8 +764,8 @@ class PROTOBUF_EXPORT TcParser final { const char* ptr, Arena* arena, SerialArena* serial_arena, ParseContext* ctx, RepeatedPtrField& field); - static void UnknownPackedEnum(MessageLite* msg, const TcParseTableBase* table, - uint32_t tag, int32_t enum_value); + static void AddUnknownEnum(MessageLite* msg, const TcParseTableBase* table, + uint32_t tag, int32_t enum_value); // Mini field lookup: static const TcParseTableBase::FieldEntry* FindFieldEntry( diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 1455a9e286ba..4a8e8275d701 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -1224,16 +1224,34 @@ PROTOBUF_NOINLINE const char* TcParser::FastZ64P2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback( PROTOBUF_TC_PARAM_DECL) { - // If we know we want to put this field directly into the unknown field set, - // then we can skip the call to MiniParse and directly call table->fallback. - // However, we first have to update `data` to contain the decoded tag. + // Skip MiniParse/fallback and insert the element directly into the unknown + // field set. We also normalize the value into an int32 as we do for known + // enum values. uint32_t tag; ptr = ReadTag(ptr, &tag); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } - data.data = tag; - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + uint64_t tmp; + ptr = ParseVarint(ptr, &tmp); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + AddUnknownEnum(msg, table, tag, static_cast(tmp)); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::MpUnknownEnumFallback( + PROTOBUF_TC_PARAM_DECL) { + // Like FastUnknownEnumFallback, but with the Mp ABI. + uint32_t tag = data.tag(); + uint64_t tmp; + ptr = ParseVarint(ptr, &tmp); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + AddUnknownEnum(msg, table, tag, static_cast(tmp)); + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } template @@ -1325,9 +1343,10 @@ const TcParser::UnknownFieldOps& TcParser::GetUnknownFieldOps( return *reinterpret_cast(ptr); } -PROTOBUF_NOINLINE void TcParser::UnknownPackedEnum( - MessageLite* msg, const TcParseTableBase* table, uint32_t tag, - int32_t enum_value) { +PROTOBUF_NOINLINE void TcParser::AddUnknownEnum(MessageLite* msg, + const TcParseTableBase* table, + uint32_t tag, + int32_t enum_value) { GetUnknownFieldOps(table).write_varint(msg, tag >> 3, enum_value); } @@ -1351,7 +1370,7 @@ const char* TcParser::PackedEnum(PROTOBUF_TC_PARAM_DECL) { const TcParseTableBase::FieldAux aux = *table->field_aux(data.aux_idx()); return ctx->ReadPackedVarint(ptr, [=](int32_t value) { if (!EnumIsValidAux(value, xform_val, aux)) { - UnknownPackedEnum(msg, table, FastDecodeTag(saved_tag), value); + AddUnknownEnum(msg, table, FastDecodeTag(saved_tag), value); } else { field->Add(value); } @@ -1498,7 +1517,7 @@ const char* TcParser::PackedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) { return ctx->ReadPackedVarint(ptr, [=](int32_t v) { if (PROTOBUF_PREDICT_FALSE(min > v || v > max)) { - UnknownPackedEnum(msg, table, FastDecodeTag(saved_tag), v); + AddUnknownEnum(msg, table, FastDecodeTag(saved_tag), v); } else { field->Add(v); } @@ -2021,7 +2040,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) { if (is_validated_enum) { if (!EnumIsValidAux(tmp, xform_val, *table->field_aux(&entry))) { ptr = ptr2; - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MpUnknownEnumFallback(PROTOBUF_TC_PARAM_PASS); } } else if (is_zigzag) { tmp = WireFormatLite::ZigZagDecode32(static_cast(tmp)); @@ -2099,7 +2118,8 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint( if (is_validated_enum) { if (!EnumIsValidAux(tmp, xform_val, *table->field_aux(&entry))) { ptr = ptr2; - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + PROTOBUF_MUSTTAIL return MpUnknownEnumFallback( + PROTOBUF_TC_PARAM_PASS); } } else if (is_zigzag) { tmp = WireFormatLite::ZigZagDecode32(tmp); @@ -2163,7 +2183,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) { const TcParseTableBase::FieldAux aux = *table->field_aux(entry.aux_idx); return ctx->ReadPackedVarint(ptr, [=](int32_t value) { if (!EnumIsValidAux(value, xform_val, aux)) { - UnknownPackedEnum(msg, table, data.tag(), value); + AddUnknownEnum(msg, table, data.tag(), value); } else { field->Add(value); } diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 8cfe8c1d5abc..f5e330e0a498 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -61,6 +61,7 @@ #include "absl/strings/substitute.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" #include "google/protobuf/generated_message_reflection.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" @@ -1310,18 +1311,62 @@ std::string EncodeOverlongEnum(int number, bool use_packed) { } } +std::string EncodeInt32Value(int number, int32_t value, + int non_canonical_bytes) { + uint8_t buf[100]; + uint8_t* p = buf; + + p = internal::WireFormatLite::WriteInt32ToArray(number, value, p); + p = AddNonCanonicalBytes(SkipTag(buf), p, non_canonical_bytes); + return std::string(buf, p); +} + +std::string EncodeInt64Value(int number, int64_t value, int non_canonical_bytes, + bool use_packed = false) { + uint8_t buf[100]; + uint8_t* p = buf; + + if (use_packed) { + p = internal::WireFormatLite::WriteInt64NoTagToArray(value, p); + p = AddNonCanonicalBytes(buf, p, non_canonical_bytes); + + std::string payload(buf, p); + p = buf; + p = internal::WireFormatLite::WriteStringToArray(number, payload, p); + return std::string(buf, p); + + } else { + p = internal::WireFormatLite::WriteInt64ToArray(number, value, p); + p = AddNonCanonicalBytes(SkipTag(buf), p, non_canonical_bytes); + return std::string(buf, p); + } +} + std::string EncodeOtherField() { UNITTEST::EnumParseTester obj; obj.set_other_field(1); return obj.SerializeAsString(); } +template +static std::vector GetFields() { + auto* descriptor = T::descriptor(); + std::vector fields; + for (int i = 0; i < descriptor->field_count(); ++i) { + fields.push_back(descriptor->field(i)); + } + for (int i = 0; i < descriptor->extension_count(); ++i) { + fields.push_back(descriptor->extension(i)); + } + return fields; +} + TEST(MESSAGE_TEST_NAME, TestEnumParsers) { UNITTEST::EnumParseTester obj; const auto other_field = EncodeOtherField(); - // Encode a boolean field for many different cases and verify that it can be + // Encode an enum field for many different cases and verify that it can be // parsed as expected. // There are: // - optional/repeated/packed fields @@ -1331,6 +1376,9 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { // - label combinations to trigger different parsers: sequential, small // sequential, non-validated. + const std::vector fields = + GetFields(); + constexpr int kInvalidValue = 0x900913; auto* ref = obj.GetReflection(); auto* descriptor = obj.descriptor(); @@ -1347,8 +1395,7 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { continue; } SCOPED_TRACE(add_garbage_bits); - for (int i = 0; i < descriptor->field_count(); ++i) { - const auto* field = descriptor->field(i); + for (auto field : fields) { if (field->name() == "other_field") continue; if (!field->is_repeated() && use_packed) continue; SCOPED_TRACE(field->full_name()); @@ -1421,6 +1468,52 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { } } +TEST(MESSAGE_TEST_NAME, TestEnumParserForUnknownEnumValue) { + DynamicMessageFactory factory; + std::unique_ptr dynamic( + factory.GetPrototype(UNITTEST::EnumParseTester::descriptor())->New()); + + UNITTEST::EnumParseTester non_dynamic; + + // For unknown enum values, for consistency we must include the + // int32_t enum value in the unknown field set, which might not be exactly the + // same as the input. + auto* descriptor = non_dynamic.descriptor(); + + const std::vector fields = + GetFields(); + + for (bool use_dynamic : {false, true}) { + SCOPED_TRACE(use_dynamic); + for (auto field : fields) { + if (field->name() == "other_field") continue; + SCOPED_TRACE(field->full_name()); + for (bool use_packed : {false, true}) { + SCOPED_TRACE(use_packed); + if (!field->is_repeated() && use_packed) continue; + + // -2 is an invalid enum value on all the tests here. + // We will encode -2 as a positive int64 that is equivalent to + // int32_t{-2} when truncated. + constexpr int64_t minus_2_non_canonical = + static_cast(static_cast(int32_t{-2})); + static_assert(minus_2_non_canonical != -2, ""); + std::string encoded = EncodeInt64Value( + field->number(), minus_2_non_canonical, 0, use_packed); + + auto& obj = use_dynamic ? *dynamic : non_dynamic; + ASSERT_TRUE(obj.ParseFromString(encoded)); + + auto& unknown = obj.GetReflection()->GetUnknownFields(obj); + ASSERT_EQ(unknown.field_count(), 1); + EXPECT_EQ(unknown.field(0).number(), field->number()); + EXPECT_EQ(unknown.field(0).type(), unknown.field(0).TYPE_VARINT); + EXPECT_EQ(unknown.field(0).varint(), int64_t{-2}); + } + } + } +} + std::string EncodeBoolValue(int number, bool value, int non_canonical_bytes) { uint8_t buf[100]; uint8_t* p = buf; @@ -1443,6 +1536,9 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) { // - canonical and non-canonical encodings of the varint // - last vs not last field + const std::vector fields = + GetFields(); + auto* ref = obj.GetReflection(); auto* descriptor = obj.descriptor(); for (bool use_tail_field : {false, true}) { @@ -1456,8 +1552,7 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) { continue; } SCOPED_TRACE(add_garbage_bits); - for (int i = 0; i < descriptor->field_count(); ++i) { - const auto* field = descriptor->field(i); + for (auto field : fields) { if (field->name() == "other_field") continue; SCOPED_TRACE(field->full_name()); for (bool value : {false, true}) { @@ -1492,16 +1587,6 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) { } } -std::string EncodeInt32Value(int number, int32_t value, - int non_canonical_bytes) { - uint8_t buf[100]; - uint8_t* p = buf; - - p = internal::WireFormatLite::WriteInt32ToArray(number, value, p); - p = AddNonCanonicalBytes(SkipTag(buf), p, non_canonical_bytes); - return std::string(buf, p); -} - TEST(MESSAGE_TEST_NAME, TestInt32Parsers) { UNITTEST::Int32ParseTester obj; @@ -1515,6 +1600,9 @@ TEST(MESSAGE_TEST_NAME, TestInt32Parsers) { // - canonical and non-canonical encodings of the varint // - last vs not last field + const std::vector fields = + GetFields(); + auto* ref = obj.GetReflection(); auto* descriptor = obj.descriptor(); for (bool use_tail_field : {false, true}) { @@ -1528,8 +1616,7 @@ TEST(MESSAGE_TEST_NAME, TestInt32Parsers) { continue; } SCOPED_TRACE(add_garbage_bits); - for (int i = 0; i < descriptor->field_count(); ++i) { - const auto* field = descriptor->field(i); + for (auto field : fields) { if (field->name() == "other_field") continue; SCOPED_TRACE(field->full_name()); for (int32_t value : {1, 0, -1, (std::numeric_limits::min)(), @@ -1565,16 +1652,6 @@ TEST(MESSAGE_TEST_NAME, TestInt32Parsers) { } } -std::string EncodeInt64Value(int number, int64_t value, - int non_canonical_bytes) { - uint8_t buf[100]; - uint8_t* p = buf; - - p = internal::WireFormatLite::WriteInt64ToArray(number, value, p); - p = AddNonCanonicalBytes(SkipTag(buf), p, non_canonical_bytes); - return std::string(buf, p); -} - TEST(MESSAGE_TEST_NAME, TestInt64Parsers) { UNITTEST::Int64ParseTester obj; @@ -1588,6 +1665,9 @@ TEST(MESSAGE_TEST_NAME, TestInt64Parsers) { // - canonical and non-canonical encodings of the varint // - last vs not last field + const std::vector fields = + GetFields(); + auto* ref = obj.GetReflection(); auto* descriptor = obj.descriptor(); for (bool use_tail_field : {false, true}) { @@ -1601,8 +1681,7 @@ TEST(MESSAGE_TEST_NAME, TestInt64Parsers) { continue; } SCOPED_TRACE(add_garbage_bits); - for (int i = 0; i < descriptor->field_count(); ++i) { - const auto* field = descriptor->field(i); + for (auto field : fields) { if (field->name() == "other_field") continue; SCOPED_TRACE(field->full_name()); for (int64_t value : {int64_t{1}, int64_t{0}, int64_t{-1}, @@ -1748,6 +1827,9 @@ TEST(MESSAGE_TEST_NAME, TestRepeatedStringParsers) { const auto* const descriptor = UNITTEST::StringParseTester::descriptor(); + const std::vector fields = + GetFields(); + static const size_t sso_capacity = std::string().capacity(); if (sso_capacity == 0) GTEST_SKIP(); // SSO, !SSO, and off-by-one just in case @@ -1755,8 +1837,7 @@ TEST(MESSAGE_TEST_NAME, TestRepeatedStringParsers) { {sso_capacity - 1, sso_capacity, sso_capacity + 1, sso_capacity + 2}) { SCOPED_TRACE(size); const std::string value = sample.substr(0, size); - for (int i = 0; i < descriptor->field_count(); ++i) { - const auto* field = descriptor->field(i); + for (auto field : fields) { SCOPED_TRACE(field->full_name()); const auto encoded = EncodeStringValue(field->number(), sample) + EncodeStringValue(field->number(), value); diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 0169fe7828a8..e3f8ae0c6718 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -1170,7 +1170,7 @@ PROTOBUF_NODISCARD const char* PackedEnumParser(void* object, const char* ptr, InternalMetadata* metadata, int field_num) { return ctx->ReadPackedVarint( - ptr, [object, is_valid, metadata, field_num](uint64_t val) { + ptr, [object, is_valid, metadata, field_num](int32_t val) { if (is_valid(val)) { static_cast*>(object)->Add(val); } else { @@ -1185,7 +1185,7 @@ PROTOBUF_NODISCARD const char* PackedEnumParserArg( bool (*is_valid)(const void*, int), const void* data, InternalMetadata* metadata, int field_num) { return ctx->ReadPackedVarint( - ptr, [object, is_valid, data, metadata, field_num](uint64_t val) { + ptr, [object, is_valid, data, metadata, field_num](int32_t val) { if (is_valid(data, val)) { static_cast*>(object)->Add(val); } else { diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index f6c57deb1b62..108572e86fde 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -1547,6 +1547,13 @@ message EnumParseTester { repeated Arbitrary packed_arbitrary_midfield = 1012 [packed = true]; repeated Arbitrary packed_arbitrary_hifield = 1000012 [packed = true]; + extensions 2000000 to max; + extend EnumParseTester { + optional Arbitrary optional_arbitrary_ext = 2000000; + repeated Arbitrary repeated_arbitrary_ext = 2000001; + repeated Arbitrary packed_arbitrary_ext = 2000002 [packed = true]; + } + // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; } @@ -1564,6 +1571,13 @@ message BoolParseTester { repeated bool packed_bool_midfield = 1003 [packed = true]; repeated bool packed_bool_hifield = 1000003 [packed = true]; + extensions 2000000 to max; + extend BoolParseTester { + optional bool optional_bool_ext = 2000000; + repeated bool repeated_bool_ext = 2000001; + repeated bool packed_bool_ext = 2000002 [packed = true]; + } + // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; } @@ -1579,6 +1593,13 @@ message Int32ParseTester { repeated int32 packed_int32_midfield = 1003 [packed = true]; repeated int32 packed_int32_hifield = 1000003 [packed = true]; + extensions 2000000 to max; + extend Int32ParseTester { + optional int32 optional_int32_ext = 2000000; + repeated int32 repeated_int32_ext = 2000001; + repeated int32 packed_int32_ext = 2000002 [packed = true]; + } + // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; } @@ -1594,6 +1615,13 @@ message Int64ParseTester { repeated int64 packed_int64_midfield = 1003 [packed = true]; repeated int64 packed_int64_hifield = 1000003 [packed = true]; + extensions 2000000 to max; + extend Int64ParseTester { + optional int64 optional_int64_ext = 2000000; + repeated int64 repeated_int64_ext = 2000001; + repeated int64 packed_int64_ext = 2000002 [packed = true]; + } + // An arbitrary field we can append to to break the runs of repeated fields. optional int32 other_field = 99; } @@ -1617,6 +1645,12 @@ message StringParseTester { repeated string repeated_string_lowfield = 2; repeated string repeated_string_midfield = 1002; repeated string repeated_string_hifield = 1000002; + + extensions 2000000 to max; + extend StringParseTester { + optional string optional_string_ext = 2000000; + repeated string repeated_string_ext = 2000001; + } } message BadFieldNames{ diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index e0c18bb00cf6..fe62d1c622da 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -874,7 +874,7 @@ const char* WireFormat::_InternalParseAndMergeField( ptr = internal::PackedEnumParser(rep_enum, ptr, ctx); } else { return ctx->ReadPackedVarint( - ptr, [rep_enum, field, reflection, msg](uint64_t val) { + ptr, [rep_enum, field, reflection, msg](int32_t val) { if (field->enum_type()->FindValueByNumber(val) != nullptr) { rep_enum->Add(val); } else { From b3d1e99242f104c7ef1aee1bff05bf187bd84378 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 24 Feb 2023 20:57:11 +0000 Subject: [PATCH 108/463] Auto-generate files after cl/512140469 --- src/google/protobuf/api.pb.cc | 4 ++-- src/google/protobuf/struct.pb.cc | 2 +- src/google/protobuf/type.pb.cc | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index ed63bbb65397..649d48dc6054 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -436,7 +436,7 @@ const char* Api::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Syntax syntax = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 56)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_syntax(static_cast<::PROTOBUF_NAMESPACE_ID::Syntax>(val)); } else { @@ -865,7 +865,7 @@ const char* Method::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Syntax syntax = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 56)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_syntax(static_cast<::PROTOBUF_NAMESPACE_ID::Syntax>(val)); } else { diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index a5fa0b8c0c02..47e930ee1ebc 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -618,7 +618,7 @@ const char* Value::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.NullValue null_value = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 8)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_null_value(static_cast<::PROTOBUF_NAMESPACE_ID::NullValue>(val)); } else { diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index c608689d2277..49c5c37c9735 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -599,7 +599,7 @@ const char* Type::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Syntax syntax = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 48)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_syntax(static_cast<::PROTOBUF_NAMESPACE_ID::Syntax>(val)); } else { @@ -965,7 +965,7 @@ const char* Field::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Field.Kind kind = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 8)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::PROTOBUF_NAMESPACE_ID::Field_Kind>(val)); } else { @@ -975,7 +975,7 @@ const char* Field::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Field.Cardinality cardinality = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 16)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_cardinality(static_cast<::PROTOBUF_NAMESPACE_ID::Field_Cardinality>(val)); } else { @@ -1497,7 +1497,7 @@ const char* Enum::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { // .google.protobuf.Syntax syntax = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::uint8_t>(tag) == 40)) { - ::uint32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); + ::int32_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); _internal_set_syntax(static_cast<::PROTOBUF_NAMESPACE_ID::Syntax>(val)); } else { From df2aac3b0fecd4e0339fd04e6e1576ba0022ce53 Mon Sep 17 00:00:00 2001 From: Martijn Vels Date: Fri, 24 Feb 2023 13:35:24 -0800 Subject: [PATCH 109/463] Use X MACRO for generated_message_reflection.cc lookup, and pass FieldInfo This allows swapping the "name" for the enum value, which means we can make a cheap indexed table instead. PiperOrigin-RevId: 512151658 --- .../protobuf/generated_message_reflection.cc | 126 ++---------------- .../protobuf/generated_message_tctable_impl.h | 11 +- 2 files changed, 18 insertions(+), 119 deletions(-) diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index efe5d7aae211..f5c6b22292f4 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2957,124 +2957,16 @@ static uint32_t AlignTo(uint32_t v) { } static internal::TailCallParseFunc GetFastParseFunction( - absl::string_view name) { - // This list must be synchronized with TcParser. - // Missing entries are replaced with MiniParse in opt mode to avoid runtime - // failures. It check-fails in debug mode. + const internal::TailCallTableInfo::FastFieldInfo& field_info) { +#define PROTOBUF_TC_PARSE_FUNCTION_X(value) \ + {"::_pbi::TcParser::" #value, internal::TcParser::value}, static const auto* const map = new absl::flat_hash_map{ - {"::_pbi::TcParser::FastF32S1", internal::TcParser::FastF32S1}, - {"::_pbi::TcParser::FastF32S2", internal::TcParser::FastF32S2}, - {"::_pbi::TcParser::FastF32R1", internal::TcParser::FastF32R1}, - {"::_pbi::TcParser::FastF32R2", internal::TcParser::FastF32R2}, - {"::_pbi::TcParser::FastF32P1", internal::TcParser::FastF32P1}, - {"::_pbi::TcParser::FastF32P2", internal::TcParser::FastF32P2}, - {"::_pbi::TcParser::FastF64S1", internal::TcParser::FastF64S1}, - {"::_pbi::TcParser::FastF64S2", internal::TcParser::FastF64S2}, - {"::_pbi::TcParser::FastF64R1", internal::TcParser::FastF64R1}, - {"::_pbi::TcParser::FastF64R2", internal::TcParser::FastF64R2}, - {"::_pbi::TcParser::FastF64P1", internal::TcParser::FastF64P1}, - {"::_pbi::TcParser::FastF64P2", internal::TcParser::FastF64P2}, - {"::_pbi::TcParser::FastV8S1", internal::TcParser::FastV8S1}, - {"::_pbi::TcParser::FastV8S2", internal::TcParser::FastV8S2}, - {"::_pbi::TcParser::FastV8R1", internal::TcParser::FastV8R1}, - {"::_pbi::TcParser::FastV8R2", internal::TcParser::FastV8R2}, - {"::_pbi::TcParser::FastV8P1", internal::TcParser::FastV8P1}, - {"::_pbi::TcParser::FastV8P2", internal::TcParser::FastV8P2}, - {"::_pbi::TcParser::FastV32S1", internal::TcParser::FastV32S1}, - {"::_pbi::TcParser::FastV32S2", internal::TcParser::FastV32S2}, - {"::_pbi::TcParser::FastV32R1", internal::TcParser::FastV32R1}, - {"::_pbi::TcParser::FastV32R2", internal::TcParser::FastV32R2}, - {"::_pbi::TcParser::FastV32P1", internal::TcParser::FastV32P1}, - {"::_pbi::TcParser::FastV32P2", internal::TcParser::FastV32P2}, - {"::_pbi::TcParser::FastV64S1", internal::TcParser::FastV64S1}, - {"::_pbi::TcParser::FastV64S2", internal::TcParser::FastV64S2}, - {"::_pbi::TcParser::FastV64R1", internal::TcParser::FastV64R1}, - {"::_pbi::TcParser::FastV64R2", internal::TcParser::FastV64R2}, - {"::_pbi::TcParser::FastV64P1", internal::TcParser::FastV64P1}, - {"::_pbi::TcParser::FastV64P2", internal::TcParser::FastV64P2}, - {"::_pbi::TcParser::FastZ32S1", internal::TcParser::FastZ32S1}, - {"::_pbi::TcParser::FastZ32S2", internal::TcParser::FastZ32S2}, - {"::_pbi::TcParser::FastZ32R1", internal::TcParser::FastZ32R1}, - {"::_pbi::TcParser::FastZ32R2", internal::TcParser::FastZ32R2}, - {"::_pbi::TcParser::FastZ32P1", internal::TcParser::FastZ32P1}, - {"::_pbi::TcParser::FastZ32P2", internal::TcParser::FastZ32P2}, - {"::_pbi::TcParser::FastZ64S1", internal::TcParser::FastZ64S1}, - {"::_pbi::TcParser::FastZ64S2", internal::TcParser::FastZ64S2}, - {"::_pbi::TcParser::FastZ64R1", internal::TcParser::FastZ64R1}, - {"::_pbi::TcParser::FastZ64R2", internal::TcParser::FastZ64R2}, - {"::_pbi::TcParser::FastZ64P1", internal::TcParser::FastZ64P1}, - {"::_pbi::TcParser::FastZ64P2", internal::TcParser::FastZ64P2}, - {"::_pbi::TcParser::FastErS1", internal::TcParser::FastErS1}, - {"::_pbi::TcParser::FastErS2", internal::TcParser::FastErS2}, - {"::_pbi::TcParser::FastErR1", internal::TcParser::FastErR1}, - {"::_pbi::TcParser::FastErR2", internal::TcParser::FastErR2}, - {"::_pbi::TcParser::FastErP1", internal::TcParser::FastErP1}, - {"::_pbi::TcParser::FastErP2", internal::TcParser::FastErP2}, - {"::_pbi::TcParser::FastEr0S1", internal::TcParser::FastEr0S1}, - {"::_pbi::TcParser::FastEr0S2", internal::TcParser::FastEr0S2}, - {"::_pbi::TcParser::FastEr0R1", internal::TcParser::FastEr0R1}, - {"::_pbi::TcParser::FastEr0R2", internal::TcParser::FastEr0R2}, - {"::_pbi::TcParser::FastEr0P1", internal::TcParser::FastEr0P1}, - {"::_pbi::TcParser::FastEr0P2", internal::TcParser::FastEr0P2}, - {"::_pbi::TcParser::FastEr1S1", internal::TcParser::FastEr1S1}, - {"::_pbi::TcParser::FastEr1S2", internal::TcParser::FastEr1S2}, - {"::_pbi::TcParser::FastEr1R1", internal::TcParser::FastEr1R1}, - {"::_pbi::TcParser::FastEr1R2", internal::TcParser::FastEr1R2}, - {"::_pbi::TcParser::FastEr1P1", internal::TcParser::FastEr1P1}, - {"::_pbi::TcParser::FastEr1P2", internal::TcParser::FastEr1P2}, - {"::_pbi::TcParser::FastEvS1", internal::TcParser::FastEvS1}, - {"::_pbi::TcParser::FastEvS2", internal::TcParser::FastEvS2}, - {"::_pbi::TcParser::FastEvR1", internal::TcParser::FastEvR1}, - {"::_pbi::TcParser::FastEvR2", internal::TcParser::FastEvR2}, - {"::_pbi::TcParser::FastEvP1", internal::TcParser::FastEvP1}, - {"::_pbi::TcParser::FastEvP2", internal::TcParser::FastEvP2}, - {"::_pbi::TcParser::FastBS1", internal::TcParser::FastBS1}, - {"::_pbi::TcParser::FastBS2", internal::TcParser::FastBS2}, - {"::_pbi::TcParser::FastBR1", internal::TcParser::FastBR1}, - {"::_pbi::TcParser::FastBR2", internal::TcParser::FastBR2}, - {"::_pbi::TcParser::FastSS1", internal::TcParser::FastSS1}, - {"::_pbi::TcParser::FastSS2", internal::TcParser::FastSS2}, - {"::_pbi::TcParser::FastSR1", internal::TcParser::FastSR1}, - {"::_pbi::TcParser::FastSR2", internal::TcParser::FastSR2}, - {"::_pbi::TcParser::FastUS1", internal::TcParser::FastUS1}, - {"::_pbi::TcParser::FastUS2", internal::TcParser::FastUS2}, - {"::_pbi::TcParser::FastUR1", internal::TcParser::FastUR1}, - {"::_pbi::TcParser::FastUR2", internal::TcParser::FastUR2}, - {"::_pbi::TcParser::FastBiS1", internal::TcParser::FastBiS1}, - {"::_pbi::TcParser::FastBiS2", internal::TcParser::FastBiS2}, - {"::_pbi::TcParser::FastSiS1", internal::TcParser::FastSiS1}, - {"::_pbi::TcParser::FastSiS2", internal::TcParser::FastSiS2}, - {"::_pbi::TcParser::FastUiS1", internal::TcParser::FastUiS1}, - {"::_pbi::TcParser::FastUiS2", internal::TcParser::FastUiS2}, - {"::_pbi::TcParser::FastBcS1", internal::TcParser::FastBcS1}, - {"::_pbi::TcParser::FastBcS2", internal::TcParser::FastBcS2}, - {"::_pbi::TcParser::FastScS1", internal::TcParser::FastScS1}, - {"::_pbi::TcParser::FastScS2", internal::TcParser::FastScS2}, - {"::_pbi::TcParser::FastUcS1", internal::TcParser::FastUcS1}, - {"::_pbi::TcParser::FastUcS2", internal::TcParser::FastUcS2}, - {"::_pbi::TcParser::FastMdS1", internal::TcParser::FastMdS1}, - {"::_pbi::TcParser::FastMdS2", internal::TcParser::FastMdS2}, - {"::_pbi::TcParser::FastGdS1", internal::TcParser::FastGdS1}, - {"::_pbi::TcParser::FastGdS2", internal::TcParser::FastGdS2}, - {"::_pbi::TcParser::FastMtS1", internal::TcParser::FastMtS1}, - {"::_pbi::TcParser::FastMtS2", internal::TcParser::FastMtS2}, - {"::_pbi::TcParser::FastGtS1", internal::TcParser::FastGtS1}, - {"::_pbi::TcParser::FastGtS2", internal::TcParser::FastGtS2}, - {"::_pbi::TcParser::FastMdR1", internal::TcParser::FastMdR1}, - {"::_pbi::TcParser::FastMdR2", internal::TcParser::FastMdR2}, - {"::_pbi::TcParser::FastGdR1", internal::TcParser::FastGdR1}, - {"::_pbi::TcParser::FastGdR2", internal::TcParser::FastGdR2}, - {"::_pbi::TcParser::FastMtR1", internal::TcParser::FastMtR1}, - {"::_pbi::TcParser::FastMtR2", internal::TcParser::FastMtR2}, - {"::_pbi::TcParser::FastGtR1", internal::TcParser::FastGtR1}, - {"::_pbi::TcParser::FastGtR2", internal::TcParser::FastGtR2}, - {"::_pbi::TcParser::FastEndG1", internal::TcParser::FastEndG1}, - {"::_pbi::TcParser::FastEndG2", internal::TcParser::FastEndG2}, - }; - auto it = map->find(name); + PROTOBUF_TC_PARSE_FUNCTION_LIST}; +#undef PROTOBUF_TC_PARSE_FUNCTION_X + auto it = map->find(field_info.func_name); if (it == map->end()) { - ABSL_DLOG(FATAL) << "Failed to find function: " << name; + ABSL_DLOG(FATAL) << "Failed to find function: " << field_info.func_name; // Let's not crash in opt, just in case. // MiniParse is always a valid parser. return &internal::TcParser::MiniParse; @@ -3110,7 +3002,7 @@ void Reflection::PopulateTcParseFastEntries( *fast_entries++ = {internal::TcParser::MiniParse, {}}; } else { // No field, but still a special entry. - *fast_entries++ = {GetFastParseFunction(fast_field.func_name), + *fast_entries++ = {GetFastParseFunction(fast_field), {fast_field.coded_tag, fast_field.nonfield_info}}; } } else if (fast_field.func_name.find("TcParser::FastEv") != @@ -3121,7 +3013,7 @@ void Reflection::PopulateTcParseFastEntries( *fast_entries++ = {internal::TcParser::MiniParse, {}}; } else { *fast_entries++ = { - GetFastParseFunction(fast_field.func_name), + GetFastParseFunction(fast_field), {fast_field.coded_tag, fast_field.hasbit_idx, fast_field.aux_idx, static_cast(schema_.GetFieldOffset(fast_field.field))}}; } diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 541b4750901a..1975639c4c3f 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -281,6 +281,10 @@ inline void AlignFail(std::integral_constant, PROTOBUF_TC_PARSE_FUNCTION_X(fn##P1) \ PROTOBUF_TC_PARSE_FUNCTION_X(fn##P2) +#define PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \ + PROTOBUF_TC_PARSE_FUNCTION_X(FastEndG1) \ + PROTOBUF_TC_PARSE_FUNCTION_X(FastEndG2) + // TcParseFunction defines the set of table driven, tail call optimized parse // functions. This list currently does not include all types such as maps. // @@ -307,6 +311,7 @@ inline void AlignFail(std::integral_constant, // Gt - group width table driven parse tables // Md - message // Mt - message width table driven parse tables +// End - End group tag // // * string types can have a `c` or `i` suffix, indicating the // underlying storage type to be cord or inlined respectively. @@ -325,13 +330,14 @@ inline void AlignFail(std::integral_constant, // S - singular / optional // R - repeated // P - packed +// G - group terminated // // tag_width: // 1: single byte encoded tag // 2: two byte encoded tag // // Examples: -// FastV8S1, FastEr1P2, FastBcS1 +// FastV8S1, FastZ64S2, FastEr1P2, FastBcS1, FastMtR2, FastEndG1 // #define PROTOBUF_TC_PARSE_FUNCTION_LIST \ PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV8) \ @@ -357,7 +363,8 @@ inline void AlignFail(std::integral_constant, PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \ - PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() #define PROTOBUF_TC_PARSE_FUNCTION_X(value) k##value, enum class TcParseFunction { kNone, PROTOBUF_TC_PARSE_FUNCTION_LIST }; From 1b1ca24ae096ab96b90637da69bd70df68b2121c Mon Sep 17 00:00:00 2001 From: "Karol M. Langner" Date: Fri, 24 Feb 2023 17:27:17 -0800 Subject: [PATCH 110/463] Call setUp on super, and fix import order PiperOrigin-RevId: 512200880 --- python/google/protobuf/internal/proto_builder_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/google/protobuf/internal/proto_builder_test.py b/python/google/protobuf/internal/proto_builder_test.py index 48077b0a4bbe..d1102b81c2a4 100644 --- a/python/google/protobuf/internal/proto_builder_test.py +++ b/python/google/protobuf/internal/proto_builder_test.py @@ -33,8 +33,8 @@ import collections import unittest -from google.protobuf import descriptor_pb2 # pylint: disable=g-import-not-at-top from google.protobuf import descriptor +from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool from google.protobuf import proto_builder from google.protobuf import text_format @@ -43,6 +43,8 @@ class ProtoBuilderTest(unittest.TestCase): def setUp(self): + super().setUp() + self.ordered_fields = collections.OrderedDict([ ('foo', descriptor_pb2.FieldDescriptorProto.TYPE_INT64), ('bar', descriptor_pb2.FieldDescriptorProto.TYPE_STRING), From c0bd3dd984399d3302a8f7cf69e7c9330b362ac4 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 19:47:10 -0800 Subject: [PATCH 111/463] Implement helper function FieldDescriptor::legacy_enum_field_treated_as_closed to support legacy enum field behavior. Proto2 fields with types corresponding to proto3 enums are treated as closed today, despite the EnumDescriptor reporting that they're open. This will allow syntax users to depend on this non-conformant behavior which will be getting it's own feature in Edition zero. PiperOrigin-RevId: 512218773 --- src/google/protobuf/descriptor.h | 24 +++++++++ src/google/protobuf/descriptor_unittest.cc | 60 ++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 22bc03c74586..c3be72bd34d2 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -769,6 +769,25 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase { // parse. bool requires_utf8_validation() const; + // Determines if the given enum field is treated as closed based on legacy + // non-conformant behavior. + // + // Conformant behavior determines closedness based on the enum and + // can be queried using EnumDescriptor::is_closed(). + // + // Some runtimes currently have a quirk where non-closed enums are + // treated as closed when used as the type of fields defined in a + // `syntax = proto2;` file. This quirk is not present in all runtimes; as of + // writing, we know that: + // + // - C++, Java, and C++-based Python share this quirk. + // - UPB and UPB-based Python do not. + // - PHP and Ruby treat all enums as open regardless of declaration. + // + // Care should be taken when using this function to respect the target + // runtime's enum handling quirks. + bool legacy_enum_field_treated_as_closed() const; + // Index of this field within the message's field array, or the file or // extension scope's extensions array. int index() const; @@ -2443,6 +2462,11 @@ inline bool FieldDescriptor::requires_utf8_validation() const { file()->syntax() == FileDescriptor::SYNTAX_PROTO3; } +inline bool FieldDescriptor::legacy_enum_field_treated_as_closed() const { + return type() == TYPE_ENUM && + file()->syntax() == FileDescriptor::SYNTAX_PROTO2; +} + // To save space, index() is computed by looking at the descriptor's position // in the parent's array of children. inline int FieldDescriptor::index() const { diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 64e6b2c52ad7..c4c7e933c059 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -1120,6 +1120,66 @@ TEST_F(DescriptorTest, NeedsUtf8Check) { EXPECT_FALSE(bar3->requires_utf8_validation()); } +TEST_F(DescriptorTest, EnumFieldTreatedAsClosed) { + // Make an open enum definition. + FileDescriptorProto open_enum_file; + open_enum_file.set_name("open_enum.proto"); + open_enum_file.set_syntax("proto3"); + AddEnumValue(AddEnum(&open_enum_file, "TestEnumOpen"), "TestEnumOpen_VALUE0", + 0); + + const EnumDescriptor* open_enum = + pool_.BuildFile(open_enum_file)->enum_type(0); + EXPECT_FALSE(open_enum->is_closed()); + + // Create a message that treats enum fields as closed. + FileDescriptorProto closed_file; + closed_file.set_name("closed_enum_field.proto"); + closed_file.add_dependency("open_enum.proto"); + closed_file.add_dependency("foo.proto"); + + DescriptorProto* message = AddMessage(&closed_file, "TestClosedEnumField"); + AddField(message, "int_field", 1, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message, "open_enum", 2, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_ENUM) + ->set_type_name("TestEnumOpen"); + AddField(message, "closed_enum", 3, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_ENUM) + ->set_type_name("TestEnum"); + const Descriptor* closed_message = + pool_.BuildFile(closed_file)->message_type(0); + + EXPECT_FALSE(closed_message->FindFieldByName("int_field") + ->legacy_enum_field_treated_as_closed()); + EXPECT_TRUE(closed_message->FindFieldByName("closed_enum") + ->legacy_enum_field_treated_as_closed()); + EXPECT_TRUE(closed_message->FindFieldByName("open_enum") + ->legacy_enum_field_treated_as_closed()); +} + +TEST_F(DescriptorTest, EnumFieldTreatedAsOpen) { + FileDescriptorProto open_enum_file; + open_enum_file.set_name("open_enum.proto"); + open_enum_file.set_syntax("proto3"); + AddEnumValue(AddEnum(&open_enum_file, "TestEnumOpen"), "TestEnumOpen_VALUE0", + 0); + DescriptorProto* message = AddMessage(&open_enum_file, "TestOpenEnumField"); + AddField(message, "int_field", 1, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message, "open_enum", 2, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_ENUM) + ->set_type_name("TestEnumOpen"); + const FileDescriptor* open_enum_file_desc = pool_.BuildFile(open_enum_file); + const Descriptor* open_message = open_enum_file_desc->message_type(0); + const EnumDescriptor* open_enum = open_enum_file_desc->enum_type(0); + EXPECT_FALSE(open_enum->is_closed()); + EXPECT_FALSE(open_message->FindFieldByName("int_field") + ->legacy_enum_field_treated_as_closed()); + EXPECT_FALSE(open_message->FindFieldByName("open_enum") + ->legacy_enum_field_treated_as_closed()); +} + TEST_F(DescriptorTest, IsMap) { EXPECT_TRUE(map_->is_map()); EXPECT_FALSE(baz_->is_map()); From 343cef1bd940380ae1bf23a53f565ca93ffe6c3e Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Mon, 27 Feb 2023 07:58:59 -0800 Subject: [PATCH 112/463] Replace FileDescriptor.supportsUnknownEnumValues() with FieldDescriptor.legacyEnumFieldTreatedAsClosed() to support legacy enum field behavior. This documents Java's non-conformant behavior where closedness is determined by the field rather than the enum definition. PiperOrigin-RevId: 512627297 --- .../java/com/google/protobuf/Descriptors.java | 28 +++- .../com/google/protobuf/GeneratedMessage.java | 4 +- .../google/protobuf/GeneratedMessageV3.java | 4 +- .../google/protobuf/MessageReflection.java | 14 +- .../com/google/protobuf/DescriptorsTest.java | 124 ++++++++++++++++++ 5 files changed, 159 insertions(+), 15 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index e5b973f15804..5e458da85bdc 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -634,10 +634,6 @@ private void setProto(final FileDescriptorProto proto) { extensions[i].setProto(proto.getExtension(i)); } } - - boolean supportsUnknownEnumValue() { - return getSyntax() == Syntax.PROTO3; - } } // ================================================================= @@ -1328,6 +1324,30 @@ public EnumDescriptor getEnumType() { return enumType; } + /** + * Determines if the given enum field is treated as closed based on legacy non-conformant + * behavior. + * + *

Conformant behavior determines closedness based on the enum and can be queried using + * {@code EnumDescriptor.isClosed()}. + * + *

Some runtimes currently have a quirk where non-closed enums are treated as closed when + * used as the type of fields defined in a `syntax = proto2;` file. This quirk is not present in + * all runtimes; as of writing, we know that: + * + *

    + *
  • C++, Java, and C++-based Python share this quirk. + *
  • UPB and UPB-based Python do not. + *
  • PHP and Ruby treat all enums as open regardless of declaration. + *
+ * + *

Care should be taken when using this function to respect the target runtime's enum + * handling quirks. + */ + boolean legacyEnumFieldTreatedAsClosed() { + return getType() == Type.ENUM && getFile().getSyntax() == Syntax.PROTO2; + } + /** * Compare with another {@code FieldDescriptor}. This orders fields in "canonical" order, which * simply means ascending order by field number. {@code other} must be a field of the same type. diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index 26cc5bb759b4..e62a38218ddb 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -2646,7 +2646,7 @@ private static final class SingularEnumFieldAccessor extends SingularFieldAccess valueOfMethod = getMethodOrDie(type, "valueOf", EnumValueDescriptor.class); getValueDescriptorMethod = getMethodOrDie(type, "getValueDescriptor"); - supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); + supportUnknownEnumValue = !descriptor.legacyEnumFieldTreatedAsClosed(); if (supportUnknownEnumValue) { getValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); getValueMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); @@ -2705,7 +2705,7 @@ private static final class RepeatedEnumFieldAccessor extends RepeatedFieldAccess valueOfMethod = getMethodOrDie(type, "valueOf", EnumValueDescriptor.class); getValueDescriptorMethod = getMethodOrDie(type, "getValueDescriptor"); - supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); + supportUnknownEnumValue = !descriptor.legacyEnumFieldTreatedAsClosed(); if (supportUnknownEnumValue) { getRepeatedValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value", int.class); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index b5d3c76c5fb3..3fb6ea25861a 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -2889,7 +2889,7 @@ private static final class SingularEnumFieldAccessor extends SingularFieldAccess valueOfMethod = getMethodOrDie(type, "valueOf", EnumValueDescriptor.class); getValueDescriptorMethod = getMethodOrDie(type, "getValueDescriptor"); - supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); + supportUnknownEnumValue = !descriptor.legacyEnumFieldTreatedAsClosed(); if (supportUnknownEnumValue) { getValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); getValueMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); @@ -2950,7 +2950,7 @@ private static final class RepeatedEnumFieldAccessor extends RepeatedFieldAccess valueOfMethod = getMethodOrDie(type, "valueOf", EnumValueDescriptor.class); getValueDescriptorMethod = getMethodOrDie(type, "getValueDescriptor"); - supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); + supportUnknownEnumValue = !descriptor.legacyEnumFieldTreatedAsClosed(); if (supportUnknownEnumValue) { getRepeatedValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value", int.class); diff --git a/java/core/src/main/java/com/google/protobuf/MessageReflection.java b/java/core/src/main/java/com/google/protobuf/MessageReflection.java index 0404042019ec..06cb7cd4520f 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageReflection.java +++ b/java/core/src/main/java/com/google/protobuf/MessageReflection.java @@ -1199,10 +1199,7 @@ static boolean mergeFieldFrom( if (field.getLiteType() == WireFormat.FieldType.ENUM) { while (input.getBytesUntilLimit() > 0) { final int rawValue = input.readEnum(); - if (field.getFile().supportsUnknownEnumValue()) { - target.addRepeatedField( - field, field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue)); - } else { + if (field.legacyEnumFieldTreatedAsClosed()) { final Object value = field.getEnumType().findValueByNumber(rawValue); // If the number isn't recognized as a valid value for this enum, // add it to the unknown fields. @@ -1213,6 +1210,9 @@ static boolean mergeFieldFrom( } else { target.addRepeatedField(field, value); } + } else { + target.addRepeatedField( + field, field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue)); } } } else { @@ -1239,9 +1239,7 @@ static boolean mergeFieldFrom( } case ENUM: final int rawValue = input.readEnum(); - if (field.getFile().supportsUnknownEnumValue()) { - value = field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue); - } else { + if (field.legacyEnumFieldTreatedAsClosed()) { value = field.getEnumType().findValueByNumber(rawValue); // If the number isn't recognized as a valid value for this enum, // add it to the unknown fields. @@ -1251,6 +1249,8 @@ static boolean mergeFieldFrom( } return true; } + } else { + value = field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue); } break; default: diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index 38e547c37cc3..1cd1bb6473fe 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -289,6 +289,130 @@ public void testFieldDescriptorDefault() throws Exception { assertThat(d.findFieldByName("large_uint64").getDefaultValue()).isEqualTo(-1L); } + @Test + public void testFieldDescriptorLegacyEnumFieldTreatedAsClosed() throws Exception { + // Make an open enum definition. + FileDescriptorProto openEnumFile = + FileDescriptorProto.newBuilder() + .setName("open_enum.proto") + .setSyntax("proto3") + .addEnumType( + EnumDescriptorProto.newBuilder() + .setName("TestEnumOpen") + .addValue( + EnumValueDescriptorProto.newBuilder() + .setName("TestEnumOpen_VALUE0") + .setNumber(0) + .build()) + .build()) + .build(); + FileDescriptor openFileDescriptor = + Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); + EnumDescriptor openEnum = openFileDescriptor.getEnumTypes().get(0); + assertThat(openEnum.isClosed()).isFalse(); + + // Create a message that treats enum fields as closed. + FileDescriptorProto closedEnumFile = + FileDescriptorProto.newBuilder() + .setName("closed_enum_field.proto") + .addDependency("open_enum.proto") + .setSyntax("proto2") + .addEnumType( + EnumDescriptorProto.newBuilder() + .setName("TestEnum") + .addValue( + EnumValueDescriptorProto.newBuilder() + .setName("TestEnum_VALUE0") + .setNumber(0) + .build()) + .build()) + .addMessageType( + DescriptorProto.newBuilder() + .setName("TestClosedEnumField") + .addField( + FieldDescriptorProto.newBuilder() + .setName("int_field") + .setNumber(1) + .setType(FieldDescriptorProto.Type.TYPE_INT32) + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("open_enum") + .setNumber(2) + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setTypeName("TestEnumOpen") + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("closed_enum") + .setNumber(3) + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setTypeName("TestEnum") + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build()) + .build()) + .build(); + Descriptor closedMessage = + Descriptors.FileDescriptor.buildFrom( + closedEnumFile, new FileDescriptor[] {openFileDescriptor}) + .getMessageTypes() + .get(0); + assertThat(closedMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()) + .isFalse(); + + assertThat(closedMessage.findFieldByName("closed_enum").legacyEnumFieldTreatedAsClosed()) + .isTrue(); + assertThat(closedMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()) + .isTrue(); + } + + @Test + public void testFieldDescriptorLegacyEnumFieldTreatedAsOpen() throws Exception { + // Make an open enum definition and message that treats enum fields as open. + FileDescriptorProto openEnumFile = + FileDescriptorProto.newBuilder() + .setName("open_enum.proto") + .setSyntax("proto3") + .addEnumType( + EnumDescriptorProto.newBuilder() + .setName("TestEnumOpen") + .addValue( + EnumValueDescriptorProto.newBuilder() + .setName("TestEnumOpen_VALUE0") + .setNumber(0) + .build()) + .build()) + .addMessageType( + DescriptorProto.newBuilder() + .setName("TestOpenEnumField") + .addField( + FieldDescriptorProto.newBuilder() + .setName("int_field") + .setNumber(1) + .setType(FieldDescriptorProto.Type.TYPE_INT32) + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("open_enum") + .setNumber(2) + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setTypeName("TestEnumOpen") + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .build()) + .build()) + .build(); + FileDescriptor openEnumFileDescriptor = + Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); + Descriptor openMessage = openEnumFileDescriptor.getMessageTypes().get(0); + EnumDescriptor openEnum = openEnumFileDescriptor.findEnumTypeByName("TestEnumOpen"); + assertThat(openEnum.isClosed()).isFalse(); + assertThat(openMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()).isFalse(); + assertThat(openMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()).isFalse(); + } + @Test public void testEnumDescriptor() throws Exception { EnumDescriptor enumType = ForeignEnum.getDescriptor(); From 10d185f7bddf263b69182930279dd03ee4b2ec4f Mon Sep 17 00:00:00 2001 From: Dimitris Koutsogiorgas Date: Mon, 27 Feb 2023 08:08:47 -0800 Subject: [PATCH 113/463] [ObjC] Add ModuleForFile helper method in ImportWriter (#12012) Custom `protoc` plugins may use this method to look up the framework name for a given file if they want to generate code not related to `import`s. Closes #12012 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12012 from dnkoutso:import_writer_module_for_file 57844831d57eef876798689911cc67e87fe8127e PiperOrigin-RevId: 512629671 --- .../compiler/objectivec/import_writer.cc | 27 ++++++++++++++----- .../compiler/objectivec/import_writer.h | 3 +++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/google/protobuf/compiler/objectivec/import_writer.cc b/src/google/protobuf/compiler/objectivec/import_writer.cc index d7ef3b9a3b8a..ca0718bed6e2 100644 --- a/src/google/protobuf/compiler/objectivec/import_writer.cc +++ b/src/google/protobuf/compiler/objectivec/import_writer.cc @@ -140,15 +140,11 @@ void ImportWriter::AddFile(const FileDescriptor* file, return; } - // Lazy parse any mappings. - if (need_to_parse_mapping_file_) { - ParseFrameworkMappings(); - } + auto module_name = ModuleForFile(file); - auto proto_lookup = proto_file_to_framework_name_.find(file->name()); - if (proto_lookup != proto_file_to_framework_name_.end()) { + if (!module_name.empty()) { other_framework_imports_.emplace_back(absl::StrCat( - proto_lookup->second, "/", FilePathBasename(file), header_extension)); + module_name, "/", FilePathBasename(file), header_extension)); return; } @@ -166,6 +162,23 @@ void ImportWriter::AddRuntimeImport(const std::string& header_name) { protobuf_imports_.push_back(header_name); } +std::string ImportWriter::ModuleForFile(const FileDescriptor* file) { + ABSL_DCHECK(!IsProtobufLibraryBundledProtoFile(file)); + + // Lazy parse any mappings. + if (need_to_parse_mapping_file_) { + ParseFrameworkMappings(); + } + + auto proto_lookup = proto_file_to_framework_name_.find(file->name()); + + if (proto_lookup != proto_file_to_framework_name_.end()) { + return proto_lookup->second; + } + + return ""; +} + void ImportWriter::PrintFileImports(io::Printer* p) const { if (!other_framework_imports_.empty()) { for (const auto& header : other_framework_imports_) { diff --git a/src/google/protobuf/compiler/objectivec/import_writer.h b/src/google/protobuf/compiler/objectivec/import_writer.h index 9f5e0a02064d..1c168b1c60e2 100644 --- a/src/google/protobuf/compiler/objectivec/import_writer.h +++ b/src/google/protobuf/compiler/objectivec/import_writer.h @@ -55,6 +55,9 @@ class ImportWriter { void AddFile(const FileDescriptor* file, const std::string& header_extension); void AddRuntimeImport(const std::string& header_name); + // This can return an empty string if there is no module for the file. It also + // does not handle bundled proto files. + std::string ModuleForFile(const FileDescriptor* file); void PrintFileImports(io::Printer* p) const; void PrintRuntimeImports(io::Printer* p, bool default_cpp_symbol) const; From 77b1db01c9b300359795b301d925da28bc2b0295 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 27 Feb 2023 09:11:30 -0800 Subject: [PATCH 114/463] Add the Rust codegen to protoc After this change, the Rust codegen can be enabled via the protoc flag '--rust_out'. Due to its experimental nature we require a magic value to be provided '--rust_out=experimental-codegen=enabled:'. Make the 'RustGenerator' create an empty *.pb.rs file. PiperOrigin-RevId: 512644570 --- pkg/BUILD.bazel | 1 + src/google/protobuf/compiler/BUILD.bazel | 1 + src/google/protobuf/compiler/main.cc | 5 ++ src/google/protobuf/compiler/rust/BUILD.bazel | 1 + .../protobuf/compiler/rust/generator.cc | 87 +++++++++++++++++++ src/google/protobuf/compiler/rust/generator.h | 6 +- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/google/protobuf/compiler/rust/generator.cc diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 25513f5c088a..89a46bf4be1b 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -386,6 +386,7 @@ cc_dist_library( "//src/google/protobuf/compiler/php", "//src/google/protobuf/compiler/python", "//src/google/protobuf/compiler/ruby", + "//src/google/protobuf/compiler/rust", ], dist_deps = [ ":protobuf", diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index cfbf71487053..b1b544cede80 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -115,6 +115,7 @@ cc_library( "//src/google/protobuf/compiler/php", "//src/google/protobuf/compiler/python", "//src/google/protobuf/compiler/ruby", + "//src/google/protobuf/compiler/rust", "@com_google_absl//absl/log:initialize", ], ) diff --git a/src/google/protobuf/compiler/main.cc b/src/google/protobuf/compiler/main.cc index ea9adfb1e331..f5a064b80de0 100644 --- a/src/google/protobuf/compiler/main.cc +++ b/src/google/protobuf/compiler/main.cc @@ -39,6 +39,7 @@ #include "google/protobuf/compiler/python/generator.h" #include "google/protobuf/compiler/python/pyi_generator.h" #include "google/protobuf/compiler/ruby/ruby_generator.h" +#include "google/protobuf/compiler/rust/generator.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -112,6 +113,10 @@ int ProtobufMain(int argc, char* argv[]) { cli.RegisterGenerator("--objc_out", "--objc_opt", &objc_generator, "Generate Objective-C header and source."); + // Rust + rust::RustGenerator rust_generator; + cli.RegisterGenerator("--rust_out", &rust_generator, + "Generate Rust sources."); return cli.Run(argc, argv); } diff --git a/src/google/protobuf/compiler/rust/BUILD.bazel b/src/google/protobuf/compiler/rust/BUILD.bazel index 2a26dd5b62c7..faddffc6edc9 100644 --- a/src/google/protobuf/compiler/rust/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/BUILD.bazel @@ -7,6 +7,7 @@ load("//build_defs:cpp_opts.bzl", "COPTS") cc_library( name = "rust", + srcs = ["generator.cc"], hdrs = ["generator.h"], copts = COPTS, include_prefix = "google/protobuf/compiler/rust", diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc new file mode 100644 index 000000000000..b9e7ccbc14d6 --- /dev/null +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -0,0 +1,87 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/rust/generator.h" + +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/io/printer.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { + +bool ExperimentalRustGeneratorEnabled( + const std::vector>& options) { + static constexpr std::pair kMagicValue = + {"experimental-codegen", "enabled"}; + + return absl::c_any_of( + options, [](std::pair pair) { + return pair == kMagicValue; + }); +} + +bool RustGenerator::Generate(const FileDescriptor* file, + const std::string& parameter, + GeneratorContext* generator_context, + std::string* error) const { + std::vector> options; + ParseGeneratorParameter(parameter, &options); + + if (!ExperimentalRustGeneratorEnabled(options)) { + *error = + "The Rust codegen is highly experimental. Future versions will break " + "existing code. Use at your own risk. You can opt-in by passing " + "'experimental-codegen=enabled' to '--rust_out'."; + return false; + } + + auto basename = StripProto(file->name()); + auto outfile = absl::WrapUnique( + generator_context->Open(absl::StrCat(basename, ".pb.rs"))); + + google::protobuf::io::Printer(outfile.get()).Emit(R"cc( + // TODO: Generate Bindings + )cc"); + return true; +} + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/rust/generator.h b/src/google/protobuf/compiler/rust/generator.h index a01c6aead30d..2d08bb68e715 100644 --- a/src/google/protobuf/compiler/rust/generator.h +++ b/src/google/protobuf/compiler/rust/generator.h @@ -33,7 +33,6 @@ #include -#include "absl/log/absl_check.h" #include "google/protobuf/compiler/code_generator.h" // Must be included last. @@ -53,10 +52,7 @@ class RustGenerator final : public google::protobuf::compiler::CodeGenerator { bool Generate(const FileDescriptor* file, const std::string& parameter, GeneratorContext* generator_context, - std::string* error) const override { - ABSL_CHECK(false) << "not yet implemented"; - return false; - } + std::string* error) const override; uint64_t GetSupportedFeatures() const override { return FEATURE_PROTO3_OPTIONAL; From d97f0c223405e9a23198ee556021aad6658c104b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 27 Feb 2023 17:22:45 +0000 Subject: [PATCH 115/463] Auto-generate files after cl/512644570 --- src/file_lists.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index ba9ad86e0b05..e331ac378398 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -363,6 +363,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/pyi_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/subprocess.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/zip_writer.cc ) @@ -463,6 +464,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/pyi_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/scc.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/subprocess.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/zip_writer.h From bf0d8e238480bdbe5d0136bd3a6186b2569f8f2f Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 27 Feb 2023 11:33:16 -0800 Subject: [PATCH 116/463] Internal change to extensions. PiperOrigin-RevId: 512684685 --- src/google/protobuf/descriptor.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index efcd86e35895..a321f94f863e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3740,10 +3740,10 @@ class DescriptorBuilder { // Maximum recursion depth corresponds to 32 nested message declarations. int recursion_depth_ = 32; - void AddError(const std::string& element_name, const Message& descriptor, + void AddError(absl::string_view element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const std::string& error); - void AddError(const std::string& element_name, const Message& descriptor, + absl::string_view error); + void AddError(absl::string_view element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error); void AddRecursiveImportError(const FileDescriptorProto& proto, int from_here); @@ -3937,6 +3937,7 @@ class DescriptorBuilder { void SuggestFieldNumbers(FileDescriptor* file, const FileDescriptorProto& proto); + // Must be run only after cross-linking. void InterpretOptions(); @@ -4189,9 +4190,9 @@ DescriptorBuilder::DescriptorBuilder( DescriptorBuilder::~DescriptorBuilder() {} PROTOBUF_NOINLINE void DescriptorBuilder::AddError( - const std::string& element_name, const Message& descriptor, + absl::string_view element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const std::string& error) { + absl::string_view error) { if (error_collector_ == nullptr) { if (!had_errors_) { ABSL_LOG(ERROR) << "Invalid proto descriptor for file \"" << filename_ @@ -4206,7 +4207,7 @@ PROTOBUF_NOINLINE void DescriptorBuilder::AddError( } PROTOBUF_NOINLINE void DescriptorBuilder::AddError( - const std::string& element_name, const Message& descriptor, + absl::string_view element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error) { AddError(element_name, descriptor, location, std::string(error)); } @@ -6462,6 +6463,7 @@ void DescriptorBuilder::CrossLinkExtensionRange( } + void DescriptorBuilder::CrossLinkField(FieldDescriptor* field, const FieldDescriptorProto& proto) { if (field->options_ == nullptr) { From 97c1e2968ae9ea08ea13ce98b7fef96e14a0ae5d Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Mon, 27 Feb 2023 12:59:55 -0800 Subject: [PATCH 117/463] Make public FieldDescriptor.legacyEnumFieldTreatedAsClosed() PiperOrigin-RevId: 512708108 --- java/core/src/main/java/com/google/protobuf/Descriptors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 5e458da85bdc..d22b50447b31 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -1344,7 +1344,7 @@ public EnumDescriptor getEnumType() { *

Care should be taken when using this function to respect the target runtime's enum * handling quirks. */ - boolean legacyEnumFieldTreatedAsClosed() { + public boolean legacyEnumFieldTreatedAsClosed() { return getType() == Type.ENUM && getFile().getSyntax() == Syntax.PROTO2; } From b82232fbcbbce14aa5959755ae4791a8e45771a0 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 27 Feb 2023 15:04:44 -0800 Subject: [PATCH 118/463] Add dll export tags to Rust protoc generator PiperOrigin-RevId: 512742225 --- src/google/protobuf/compiler/rust/generator.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/compiler/rust/generator.h b/src/google/protobuf/compiler/rust/generator.h index 2d08bb68e715..5b3d0c426ccc 100644 --- a/src/google/protobuf/compiler/rust/generator.h +++ b/src/google/protobuf/compiler/rust/generator.h @@ -43,7 +43,8 @@ namespace protobuf { namespace compiler { namespace rust { -class RustGenerator final : public google::protobuf::compiler::CodeGenerator { +class PROTOC_EXPORT RustGenerator final + : public google::protobuf::compiler::CodeGenerator { public: RustGenerator() = default; RustGenerator(const RustGenerator&) = delete; From 0a480ad51a78bc1640b7b337c1c8004c63b172f2 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 27 Feb 2023 15:36:10 -0800 Subject: [PATCH 119/463] Use pwd instead of realpath for macos Closes https://github.com/protocolbuffers/protobuf/issues/10567 PiperOrigin-RevId: 512750155 --- php/generate_descriptor_protos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/php/generate_descriptor_protos.sh b/php/generate_descriptor_protos.sh index a9af40c20a8d..56d95d8c1462 100755 --- a/php/generate_descriptor_protos.sh +++ b/php/generate_descriptor_protos.sh @@ -6,11 +6,11 @@ set -e if [[ -z "${PROTOC}" ]]; then - PROTOC=$(realpath protoc) + PROTOC=$(pwd)/protoc fi if [ ! -f $PROTOC ]; then ${BAZEL:-bazel} $BAZEL_STARTUP_FLAGS build -c opt //:protoc $BAZEL_FLAGS - PROTOC=$(realpath bazel-bin/protoc) + PROTOC=$(pwd)/bazel-bin/protoc fi if test ! -e src/google/protobuf/stubs/common.h; then From dd9dd86fbca3ab5c1c7f0aa2534dc5da61530711 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Mon, 27 Feb 2023 17:40:03 -0800 Subject: [PATCH 120/463] Raise warnings for MessageFactory class usages PiperOrigin-RevId: 512779287 --- python/google/protobuf/message_factory.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index fac1165c517b..74dd4a676cb5 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py @@ -163,10 +163,9 @@ def GetPrototype(self, descriptor): Returns: A class describing the passed in descriptor. """ - # TODO(b/258832141): add this warning - # warnings.warn('MessageFactory class is deprecated. Please use ' - # 'GetMessageClass() instead of MessageFactory.GetPrototype. ' - # 'MessageFactory class will be removed after 2024.') + warnings.warn('MessageFactory class is deprecated. Please use ' + 'GetMessageClass() instead of MessageFactory.GetPrototype. ' + 'MessageFactory class will be removed after 2024.') return GetMessageClass(descriptor) def CreatePrototype(self, descriptor): @@ -181,10 +180,9 @@ def CreatePrototype(self, descriptor): Returns: A class describing the passed in descriptor. """ - # TODO(b/258832141): add this warning - # warnings.warn('Directly call CreatePrototype is wrong. Please use ' - # 'GetMessageClass() method instead. Directly use ' - # 'CreatePrototype will raise error after July 2023.') + warnings.warn('Directly call CreatePrototype is wrong. Please use ' + 'GetMessageClass() method instead. Directly use ' + 'CreatePrototype will raise error after July 2023.') return _InternalCreateMessageClass(descriptor) def GetMessages(self, files): @@ -201,11 +199,10 @@ def GetMessages(self, files): any dependent messages as well as any messages defined in the same file as a specified message. """ - # TODO(b/258832141): add this warning - # warnings.warn('MessageFactory class is deprecated. Please use ' - # 'GetMessageClassesForFiles() instead of ' - # 'MessageFactory.GetMessages(). MessageFactory class ' - # 'will be removed after 2024.') + warnings.warn('MessageFactory class is deprecated. Please use ' + 'GetMessageClassesForFiles() instead of ' + 'MessageFactory.GetMessages(). MessageFactory class ' + 'will be removed after 2024.') return GetMessageClassesForFiles(files, self.pool) From 462964ed322503af52638d54c00a0a67d7133349 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 27 Feb 2023 20:22:39 -0800 Subject: [PATCH 121/463] Fix shared object ABI exports (#5144) (#11032) On some linkers, the ABI definitions in src/libprotoc.map and friends do not match all symbols that need to be exported. This causes a linker error when trying to use the resulting shared object library: `undefined reference to 'scc_info_FileDescriptorProto_google_2fprotobuf_2fdescriptor_2eproto'` Quite a few people seem to be affected by this, see #5144 for discussion. It seems that yocto now applies this patch itself, but that doesn't help people building in other places (I encountered this on some HPC machines), so it would probably be good to apply it upstream as well. Closes #11032 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11032 from LourensVeen:issue-5144-linker-maps 3039f932aaf212bcf2f14a3f2fd00dbfb881e46b PiperOrigin-RevId: 512810858 --- src/libprotobuf-lite.map | 2 ++ src/libprotobuf.map | 2 ++ src/libprotoc.map | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/libprotobuf-lite.map b/src/libprotobuf-lite.map index 391554669481..a1853ca6cbf2 100644 --- a/src/libprotobuf-lite.map +++ b/src/libprotobuf-lite.map @@ -3,6 +3,8 @@ extern "C++" { *google*; }; + scc_info_*; + descriptor_table_*; local: *; diff --git a/src/libprotobuf.map b/src/libprotobuf.map index 391554669481..a1853ca6cbf2 100644 --- a/src/libprotobuf.map +++ b/src/libprotobuf.map @@ -3,6 +3,8 @@ extern "C++" { *google*; }; + scc_info_*; + descriptor_table_*; local: *; diff --git a/src/libprotoc.map b/src/libprotoc.map index 391554669481..a1853ca6cbf2 100644 --- a/src/libprotoc.map +++ b/src/libprotoc.map @@ -3,6 +3,8 @@ extern "C++" { *google*; }; + scc_info_*; + descriptor_table_*; local: *; From 2dc5338ea222e1f4e0357e46b702ed6a0e82aaeb Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 05:33:38 -0800 Subject: [PATCH 122/463] Introduce proto filter for inject_field_listener_events. By default all protos are included in proto instrumentation when inject_field_listener_events option is set. Adding proto instrumentation callbacks for every proto in the binary adds codesize and runtime performance penalties. This cl introduces 'protos_for_field_listener_events' option where a list of proto files can be provided (seperated by ':') for which field listener events will be injected. For example :- protos_for_field_listener_events=foo/bar/foo.proto:foo/bar.proto The field listener event will only be sent for 'foo/bar/foo.proto' and 'foo/bar.proto'. PiperOrigin-RevId: 512905094 --- src/google/protobuf/compiler/cpp/BUILD.bazel | 1 + src/google/protobuf/compiler/cpp/generator.cc | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel index bd6944025ea9..c140927fa64c 100644 --- a/src/google/protobuf/compiler/cpp/BUILD.bazel +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -85,6 +85,7 @@ cc_library( ":names", ":names_internal", "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:command_line_interface", "//src/google/protobuf/compiler:retention", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:btree", diff --git a/src/google/protobuf/compiler/cpp/generator.cc b/src/google/protobuf/compiler/cpp/generator.cc index c485cdb2a1fc..265c97dcace6 100644 --- a/src/google/protobuf/compiler/cpp/generator.cc +++ b/src/google/protobuf/compiler/cpp/generator.cc @@ -43,6 +43,7 @@ #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "google/protobuf/compiler/command_line_interface.h" #include "google/protobuf/compiler/cpp/file.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/descriptor.pb.h" @@ -161,6 +162,15 @@ bool CppGenerator::Generate(const FileDescriptor* file, } else if (key == "proto_static_reflection_h") { } else if (key == "annotate_accessor") { file_options.annotate_accessor = true; + } else if (key == "protos_for_field_listener_events") { + for (absl::string_view proto : + absl::StrSplit(value, CommandLineInterface::kPathSeparator)) { + if (proto == file->name()) { + file_options.field_listener_options.inject_field_listener_events = + true; + break; + } + } } else if (key == "inject_field_listener_events") { file_options.field_listener_options.inject_field_listener_events = true; } else if (key == "forbidden_field_listener_events") { From 7b6e8282157f0280ecb3fd9fd4c6519a7cd5afbc Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Tue, 28 Feb 2023 09:43:33 -0800 Subject: [PATCH 123/463] Remove errorprone dependency from kotlin protos. PiperOrigin-RevId: 512960332 --- java/kotlin/BUILD.bazel | 1 - java/kotlin/pom.xml | 5 ----- src/google/protobuf/compiler/java/message.cc | 6 ------ src/google/protobuf/compiler/java/message_lite.cc | 6 ------ 4 files changed, 18 deletions(-) diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index c809117afcdc..9814c7a0d3fa 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -25,7 +25,6 @@ kt_jvm_library( deps = [ ":only_for_use_in_proto_generated_code_its_generator_and_tests", "//java/lite", - "@maven//:com_google_errorprone_error_prone_annotations", ], ) diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 97b7f70c6ba0..9cc9ea03233b 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -34,11 +34,6 @@ mockito-core test - - com.google.errorprone - error_prone_annotations - 2.5.1 - com.google.guava guava diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index 1bd7449741dc..5d6a44609f97 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -1305,9 +1305,6 @@ void ImmutableMessageGenerator::GenerateKotlinMembers( "camelcase_name", name_resolver_->GetKotlinFactoryName(descriptor_)); - if (!context_->options().opensource_runtime) { - printer->Print("@com.google.errorprone.annotations.CheckReturnValue\n"); - } printer->Print( "public inline fun $camelcase_name$(block: $message_kt$.Dsl.() -> " @@ -1340,9 +1337,6 @@ void ImmutableMessageGenerator::GenerateTopLevelKotlinMembers( io::Printer* printer) const { printer->Print("@kotlin.jvm.JvmSynthetic\n"); - if (context_->options().opensource_runtime) { - printer->Print("@com.google.errorprone.annotations.CheckReturnValue\n"); - } printer->Print( "public inline fun $message$.copy(block: $message_kt$.Dsl.() -> " diff --git a/src/google/protobuf/compiler/java/message_lite.cc b/src/google/protobuf/compiler/java/message_lite.cc index 02506bace4f2..721a463c3d9b 100644 --- a/src/google/protobuf/compiler/java/message_lite.cc +++ b/src/google/protobuf/compiler/java/message_lite.cc @@ -816,9 +816,6 @@ void ImmutableMessageLiteGenerator::GenerateKotlinMembers( "camelcase_name", name_resolver_->GetKotlinFactoryName(descriptor_)); - if (!context_->options().opensource_runtime) { - printer->Print("@com.google.errorprone.annotations.CheckReturnValue\n"); - } printer->Print( "public inline fun $camelcase_name$(block: $message_kt$.Dsl.() -> " "kotlin.Unit): $message$ =\n" @@ -846,9 +843,6 @@ void ImmutableMessageLiteGenerator::GenerateKotlinMembers( void ImmutableMessageLiteGenerator::GenerateTopLevelKotlinMembers( io::Printer* printer) const { - if (!context_->options().opensource_runtime) { - printer->Print("@com.google.errorprone.annotations.CheckReturnValue\n"); - } printer->Print( "public inline fun $message$.copy(block: $message_kt$.Dsl.() -> " "kotlin.Unit): $message$ =\n" From 09677a63e9e460745f3f40a34f305bb72c9641a9 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 28 Feb 2023 10:20:13 -0800 Subject: [PATCH 124/463] Avoid running release branch staleness tests on forked repos PiperOrigin-RevId: 512971099 --- .github/workflows/staleness_check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 9b9c4e3fbec4..f04621ed1540 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -17,6 +17,7 @@ jobs: name: Test staleness ${{ matrix.os.name }} ${{ matrix.branch}} runs-on: ${{ matrix.os.value }} + if: ${{ github.event.repository.full_name == 'protocolbuffers/protobuf' || matrix.branch == 'main '}} steps: - name: Checkout ${{ matrix.branch }} uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 From d27df9288474981259aae320dbd3650028b04755 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 11:36:50 -0800 Subject: [PATCH 125/463] Internal change PiperOrigin-RevId: 512993970 --- java/core/src/test/java/com/google/protobuf/TextFormatTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index cf0061e637a2..c933504bd815 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -57,6 +57,7 @@ import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestRequired; +import protobuf_unittest.UnittestProto.TestReservedFields; import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet; import java.io.StringReader; import java.util.Arrays; From 7cc46f50de7bdf6fff20e47afa6dc9964f7c6411 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 12:59:31 -0800 Subject: [PATCH 126/463] Internal change PiperOrigin-RevId: 513015917 --- pkg/BUILD.bazel | 1 - src/google/protobuf/BUILD.bazel | 19 +++---------------- src/google/protobuf/text_format.cc | 13 +++++++++++++ 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 89a46bf4be1b..0970cceed1d2 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -362,7 +362,6 @@ cc_dist_library( "//src/google/protobuf:arena_align", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf:wkt_cc_proto", - "//src/google/protobuf:reflection_mode", "//src/google/protobuf/compiler:importer", "//src/google/protobuf/json", "//src/google/protobuf/util:delimited_message_util", diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 90dbb7b33a73..cb2835b36d3a 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -392,6 +392,7 @@ cc_library( "generated_message_tctable_gen.cc", "map_field.cc", "message.cc", + "reflection_mode.cc", "reflection_ops.cc", "service.cc", "text_format.cc", @@ -415,6 +416,7 @@ cc_library( "metadata.h", "reflection.h", "reflection_internal.h", + "reflection_mode.h", "reflection_ops.h", "service.h", "text_format.h", @@ -1314,26 +1316,11 @@ cc_test( ], ) -cc_library( - name = "reflection_mode", - srcs = ["reflection_mode.cc"], - hdrs = ["reflection_mode.h"], - include_prefix = "google/protobuf", - visibility = [ - "//:__subpackages__", - "//src/google/protobuf:__subpackages__", - ], - deps = [ - ":port_def", - ":protobuf", - ], -) - cc_test( name = "reflection_mode_test", srcs = ["reflection_mode_test.cc"], deps = [ - ":reflection_mode", + ":protobuf_nowkt", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 57ff14f29290..ac199007faf7 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -64,6 +64,7 @@ #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/map_field.h" #include "google/protobuf/message.h" +#include "google/protobuf/reflection_mode.h" #include "google/protobuf/repeated_field.h" #include "google/protobuf/unknown_field_set.h" #include "google/protobuf/wire_format_lite.h" @@ -74,6 +75,9 @@ namespace google { namespace protobuf { +using internal::ReflectionMode; +using internal::ScopedReflectionMode; + namespace { inline bool IsHexNumber(const std::string& str) { @@ -103,6 +107,8 @@ PROTOBUF_EXPORT std::atomic enable_debug_text_format_marker; } // namespace internal std::string Message::DebugString() const { + // Indicate all scoped reflection calls are from DebugString function. + ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; TextFormat::Printer printer; @@ -122,6 +128,8 @@ std::string Message::DebugString() const { } std::string Message::ShortDebugString() const { + // Indicate all scoped reflection calls are from DebugString function. + ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; TextFormat::Printer printer; @@ -146,6 +154,8 @@ std::string Message::ShortDebugString() const { } std::string Message::Utf8DebugString() const { + // Indicate all scoped reflection calls are from DebugString function. + ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; TextFormat::Printer printer; @@ -171,6 +181,9 @@ namespace internal { void PerformAbslStringify(const Message& message, absl::FunctionRef append) { + // Indicate all scoped reflection calls are from DebugString function. + ScopedReflectionMode scope(ReflectionMode::kDebugString); + // TODO(b/249835002): consider using the single line version for short TextFormat::Printer printer; printer.SetExpandAny(true); From 10cb45654555198731399811f6de744f43099305 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 13:36:26 -0800 Subject: [PATCH 127/463] Removing dependency of command_line_interface from cpp_internal. PiperOrigin-RevId: 513025937 --- src/google/protobuf/compiler/cpp/BUILD.bazel | 1 - src/google/protobuf/compiler/cpp/generator.cc | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel index c140927fa64c..bd6944025ea9 100644 --- a/src/google/protobuf/compiler/cpp/BUILD.bazel +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -85,7 +85,6 @@ cc_library( ":names", ":names_internal", "//src/google/protobuf/compiler:code_generator", - "//src/google/protobuf/compiler:command_line_interface", "//src/google/protobuf/compiler:retention", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:btree", diff --git a/src/google/protobuf/compiler/cpp/generator.cc b/src/google/protobuf/compiler/cpp/generator.cc index 265c97dcace6..ab2632007835 100644 --- a/src/google/protobuf/compiler/cpp/generator.cc +++ b/src/google/protobuf/compiler/cpp/generator.cc @@ -43,7 +43,6 @@ #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/compiler/command_line_interface.h" #include "google/protobuf/compiler/cpp/file.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/descriptor.pb.h" @@ -53,6 +52,7 @@ namespace protobuf { namespace compiler { namespace cpp { namespace { + std::string NumberedCcFileName(absl::string_view basename, int number) { return absl::StrCat(basename, ".out/", number, ".cc"); } @@ -163,8 +163,7 @@ bool CppGenerator::Generate(const FileDescriptor* file, } else if (key == "annotate_accessor") { file_options.annotate_accessor = true; } else if (key == "protos_for_field_listener_events") { - for (absl::string_view proto : - absl::StrSplit(value, CommandLineInterface::kPathSeparator)) { + for (absl::string_view proto : absl::StrSplit(value, ':')) { if (proto == file->name()) { file_options.field_listener_options.inject_field_listener_events = true; From ab6c42bc61ecd6e387441e891d10ed8f4895de14 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Tue, 28 Feb 2023 14:07:01 -0800 Subject: [PATCH 128/463] Internal change only PiperOrigin-RevId: 513034361 --- .../protobuf/compiler/python/generator.cc | 25 ++++++------------- .../protobuf/compiler/python/generator.h | 2 -- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index 58ffe0726cff..7cc7b324d79d 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -262,9 +262,9 @@ bool Generator::Generate(const FileDescriptor* file, std::unique_ptr output(context->Open(filename)); io::Printer printer(output.get(), '$'); printer.Print( - "from $internal_package$ import descriptor_pb2\n" - "\n", - "internal_package", InternalPackage()); + "from google3.net.google.protobuf.python.internal import " + "descriptor_pb2\n" + "\n"); // For static checkers, we need to explicitly assign to the symbols we // publicly export. @@ -373,11 +373,10 @@ void Generator::PrintTopBoilerplate() const { printer_->Print("import google3\n"); } printer_->Print( - "from $internal_package$ import builder as _builder\n" - "from $public_package$ import descriptor as _descriptor\n" - "from $public_package$ import descriptor_pool as _descriptor_pool\n" - "from $public_package$ import symbol_database as _symbol_database\n", - "internal_package", InternalPackage(), "public_package", PublicPackage()); + "from google.protobuf import descriptor as _descriptor\n" + "from google.protobuf import descriptor_pool as _descriptor_pool\n" + "from google.protobuf import symbol_database as _symbol_database\n" + "from google.protobuf.internal import builder as _builder\n"); printer_->Print("# @@protoc_insertion_point(imports)\n\n"); printer_->Print("_sym_db = _symbol_database.Default()\n"); @@ -1206,16 +1205,6 @@ std::string Generator::ModuleLevelServiceDescriptorName( return name; } -std::string Generator::PublicPackage() const { - return opensource_runtime_ ? "google.protobuf" - : "google3.net.google.protobuf.python.public"; -} - -std::string Generator::InternalPackage() const { - return opensource_runtime_ ? "google.protobuf.internal" - : "google3.net.google.protobuf.python.internal"; -} - // Prints standard constructor arguments serialized_start and serialized_end. // Args: // descriptor: The cpp descriptor to have a serialized reference. diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index c5182c42c049..c203cfcba8a4 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -163,8 +163,6 @@ class PROTOC_EXPORT Generator : public CodeGenerator { std::string ModuleLevelMessageName(const Descriptor& descriptor) const; std::string ModuleLevelServiceDescriptorName( const ServiceDescriptor& descriptor) const; - std::string PublicPackage() const; - std::string InternalPackage() const; template void PrintSerializedPbInterval(const DescriptorT& descriptor, From 723bd4c3c1a51ccf7e9726453f0b710223c4b583 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 28 Feb 2023 14:07:48 -0800 Subject: [PATCH 129/463] Modify release artifacts for protoc to statically link system libraries. Closes #12063 PiperOrigin-RevId: 513034570 --- .bazelrc | 7 ------- .github/workflows/test_cpp.yml | 10 +++++----- .github/workflows/test_csharp.yml | 2 +- .github/workflows/test_java.yml | 6 +++--- .github/workflows/test_php.yml | 4 +++- .github/workflows/test_python.yml | 4 ++-- .github/workflows/test_ruby.yml | 5 +++-- BUILD.bazel | 18 ++++++++++++++++-- build_defs/cpp_opts.bzl | 7 ------- ci/common.bazelrc | 8 ++++---- pkg/BUILD.bazel | 2 +- src/google/protobuf/compiler/BUILD.bazel | 4 ++-- 12 files changed, 40 insertions(+), 37 deletions(-) diff --git a/.bazelrc b/.bazelrc index de30d6b48c20..fb29fe10d926 100644 --- a/.bazelrc +++ b/.bazelrc @@ -17,13 +17,6 @@ build:msan --copt=-fsanitize-memory-use-after-dtor build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 build:msan --copt=-DMEMORY_SANITIZER=1 -# Use our instrumented LLVM libc++ in Kokoro. -build:kokoro-msan --config=msan -build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib -build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib -build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ - - build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread build:tsan --copt=-DTHREAD_SANITIZER=1 diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 1a63692a59dc..b107eafea327 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -17,25 +17,25 @@ jobs: - { name: Optimized, flags: --config=opt } - { name: Debug, flags: --config=dbg } - { name: ASAN, flags: --config=asan } - - { name: MSAN, flags: --config=kokoro-msan } + - { name: MSAN, flags: --config=docker-msan } - { name: TSAN, flags: --config=tsan } - { name: UBSAN, flags: --config=ubsan } - { name: No-RTTI, flags: --cxxopt=-fno-rtti } include: # Set defaults - - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807 + - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9 - targets: //pkg/... //src/... @com_google_protobuf_examples//... # Override cases with custom images - config: { name: "TCMalloc" } - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea" targets: "//src/..." - config: { name: "aarch64" } targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d" - config: { name: "Bazel4" } targets: "//src/..." - image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664" + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-6361b3a6e5c97e9951d03a4de28542fc45f1adab" name: Linux ${{ matrix.config.name }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index d85867f3e9fb..c9e651eaf2bb 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -20,7 +20,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: csharp_linux bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index bb6ea697e90a..da66c4623759 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -16,15 +16,15 @@ jobs: include: - name: OpenJDK 11 version: '11' - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab targets: //java/... //java/internal:java_version - name: OpenJDK 17 version: '17' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-65526ea124d1034eac33e7c37cc6d65c5bef054f + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-508417e5215994ade7585d28ba3aad681a25fa5d targets: //java/... //java/internal:java_version - name: aarch64 version: 'aarch64' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test name: Linux ${{ matrix.name }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 4b9a6d02efa4..e3dbe4fb0239 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -45,7 +45,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} @@ -80,6 +80,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 @@ -111,6 +112,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 13e5575b24c2..a94018c99aa2 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -27,7 +27,7 @@ jobs: targets: //python/... //python:aarch64_test # TODO(b/262628111) Enable this once conformance tests are fixed. flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d name: Linux ${{ matrix.type }} ${{ matrix.version }} runs-on: ubuntu-latest @@ -39,7 +39,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 0f9392f526dd..33906f5e72e3 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -34,7 +34,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION @@ -52,6 +52,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 @@ -124,7 +125,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }} bash: > diff --git a/BUILD.bazel b/BUILD.bazel index 69a1ba96c86a..bc229ddf671f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") load("@rules_java//java:defs.bzl", "java_lite_proto_library", "java_proto_library") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library") licenses(["notice"]) @@ -168,7 +168,21 @@ alias( cc_binary( name = "protoc", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, + visibility = ["//visibility:public"], + deps = ["//src/google/protobuf/compiler:protoc_lib"], +) + +cc_binary( + name = "protoc_static", + copts = COPTS, + linkopts = LINK_OPTS, + features = select({ + # This isn't possible on mac because there is no static library for lcrt0.o + "@platforms//os:osx": [], + # When cross-compiling we need to statically link all C++ libraries. + "//conditions:default": ["fully_static_link"], + }), visibility = ["//visibility:public"], deps = ["//src/google/protobuf/compiler:protoc_lib"], ) diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index 33e157875bd1..f95c3e257902 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -41,10 +41,3 @@ LINK_OPTS = select({ "-lm", ], }) - -# When cross-compiling for Windows we need to statically link pthread and the C++ library. -PROTOC_LINK_OPTS = select({ - "//build_defs:config_win32": ["-static"], - "//build_defs:config_win64": ["-static"], - "//conditions:default": [], -}) diff --git a/ci/common.bazelrc b/ci/common.bazelrc index 421bcc093469..e5345cfd691d 100644 --- a/ci/common.bazelrc +++ b/ci/common.bazelrc @@ -19,10 +19,10 @@ build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 build:msan --copt=-DMEMORY_SANITIZER=1 # Use our instrumented LLVM libc++ in Kokoro. -build:kokoro-msan --config=msan -build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib -build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib -build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ +build:docker-msan --config=msan +build:docker-msan --linkopt=-L/opt/libcxx_msan/lib --linkopt=-lc++abi +build:docker-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib +build:docker-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 0970cceed1d2..00f450238ab0 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -59,7 +59,7 @@ Please refer to our official github site for more installation instructions: pkg_files( name = "protoc_files", - srcs = ["//:protoc"], + srcs = ["//:protoc_static"], attributes = pkg_attributes(mode = "0555"), prefix = "bin/", visibility = ["//visibility:private"], diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index b1b544cede80..b8d9e2d3db40 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -11,7 +11,7 @@ load( ) load("@rules_proto//proto:defs.bzl", "proto_library") load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") proto_library( name = "plugin_proto", @@ -123,7 +123,7 @@ cc_library( cc_binary( name = "protoc_nowkt", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, visibility = [ "//src/google/protobuf:__pkg__", ], From 2b462ae58ddc5b97c62fd78cc76ca2be62aeed0e Mon Sep 17 00:00:00 2001 From: Joshua Humphries <2035234+jhump@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:50:30 -0800 Subject: [PATCH 130/463] Fix trailing comment attribution when file has no final newline (#12082) Fixes #12081. The issue was the call to `MaybeDetachComment`: the conditional assumed that there was a next token, which was on the same line as the previous one, making attribution unclear. However, if there is no next token, we should not detach. The actual fix is a one-liner. The rest of this PR is updates to the tests to verify this behavior under a handful of scenarios. Closes #12082 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12082 from jhump:jh/fix-trailing-comment-attribution 767e41cb05ba6e176977e86fe33e9aad090f83f6 PiperOrigin-RevId: 513046172 --- src/google/protobuf/io/tokenizer.cc | 3 +- src/google/protobuf/io/tokenizer_unittest.cc | 48 ++++++++++++++++---- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index 53229f8e67e0..d75dfabf195a 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -942,7 +942,8 @@ bool Tokenizer::NextWithComments(std::string* prev_trailing_comments, // makes no sense to attach a comment to the following token. collector.Flush(); } - if (prev_line == line_ || trailing_comment_end_line == line_) { + if (result && + (prev_line == line_ || trailing_comment_end_line == line_)) { // When previous token and this one are on the same line, or // even if a multi-line trailing comment ends on the same line // as this token, it's unclear to what token the comment diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 2654acba69ca..b956c18f3947 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -643,6 +643,7 @@ inline std::ostream& operator<<(std::ostream& out, return out << absl::CEscape(test_case.input); } +// clang-format off DocCommentCase kDocCommentCases[] = { {"prev next", @@ -650,6 +651,18 @@ DocCommentCase kDocCommentCases[] = { {}, ""}, + {"prev // no next token\n", + + " no next token\n", + {}, + ""}, + + {"prev // no next token and no trailing newline", + + " no next token and no trailing newline", + {}, + ""}, + {"prev /* detached */ next", "", @@ -780,7 +793,7 @@ DocCommentCase kDocCommentCases[] = { prev /* a single block comment that spans multiple lines is detached if it ends - on the same line as next */ next" + on the same line as next */ next )pb", "", @@ -791,7 +804,7 @@ DocCommentCase kDocCommentCases[] = { ""}, {R"pb( - prev /* trailing */ /* leading */ next" + prev /* trailing */ /* leading */ next )pb", " trailing ", @@ -802,13 +815,26 @@ DocCommentCase kDocCommentCases[] = { prev /* multi-line trailing */ /* an oddly placed detached */ /* an oddly - placed leading */ next" + placed leading */ next )pb", " multi-line\ntrailing ", {" an oddly\nplaced detached "}, " an oddly\nplaced leading "}, + + {R"pb( + prev // trailing with newline + // detached + /* another detached */ + // leading but no next token to attach it to + )pb", + + " trailing with newline\n", + {" detached\n", " another detached ", + " leading but no next token to attach it to\n"}, + ""}, }; +// clang-format on TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { // Set up the tokenizer. @@ -822,8 +848,8 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { kDocCommentCases_case.input.size(), kBlockSizes_case); Tokenizer tokenizer2(&input2, &error_collector); - tokenizer.Next(); - tokenizer2.Next(); + EXPECT_TRUE(tokenizer.Next()); + EXPECT_TRUE(tokenizer2.Next()); EXPECT_EQ("prev", tokenizer.current().text); EXPECT_EQ("prev", tokenizer2.current().text); @@ -831,11 +857,13 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { std::string prev_trailing_comments; std::vector detached_comments; std::string next_leading_comments; - tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments, - &next_leading_comments); - tokenizer2.NextWithComments(NULL, NULL, NULL); - EXPECT_EQ("next", tokenizer.current().text); - EXPECT_EQ("next", tokenizer2.current().text); + bool has_next = tokenizer.NextWithComments( + &prev_trailing_comments, &detached_comments, &next_leading_comments); + EXPECT_EQ(has_next, tokenizer2.NextWithComments(nullptr, nullptr, nullptr)); + if (has_next) { + EXPECT_EQ("next", tokenizer.current().text); + EXPECT_EQ("next", tokenizer2.current().text); + } EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments, prev_trailing_comments); From e67136d289e6cf4265e2763dd77216940c400ac9 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 28 Feb 2023 15:03:20 -0800 Subject: [PATCH 131/463] Version protoc according to the compiler version number. Previously we were using the C++ runtime version, which includes an extra major version number. See #12064 PiperOrigin-RevId: 513049588 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c068aae43cfc..cc574f90a7a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ message(STATUS "${protobuf_VERSION_PRERELEASE}") # Package version set(protobuf_VERSION - "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}") + "${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}") if(protobuf_VERSION_PRERELEASE) set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}") From 33958a70b0d4d969a9055bd9dc3f20b1b52aed9c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 16:40:16 -0800 Subject: [PATCH 132/463] Internal change. PiperOrigin-RevId: 513074072 --- src/google/protobuf/generated_message_tctable_lite.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 4a8e8275d701..64d955c1b73b 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -2242,10 +2242,6 @@ PROTOBUF_NOINLINE const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) { } const uint16_t xform_val = type_card & field_layout::kTvMask; const uint16_t rep = type_card & field_layout::kRepMask; - if (rep == field_layout::kRepIString) { - // TODO(b/198211897): support InilnedStringField. - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); - } // Mark the field as present: const bool is_oneof = card == field_layout::kFcOneof; @@ -2275,6 +2271,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) { } case field_layout::kRepIString: { + ABSL_DCHECK(false); break; } } From 8a33295b8ec2111197908d9980cf1e60f9823cd4 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 28 Feb 2023 21:58:52 -0800 Subject: [PATCH 133/463] Internal change PiperOrigin-RevId: 513133876 --- src/google/protobuf/arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index e9e7a15d4e4d..0ec9cf312935 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -552,7 +552,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); - if (arena == nullptr) { + if (PROTOBUF_PREDICT_FALSE(arena == nullptr)) { // Generated arena constructor T(Arena*) is protected. Call via // InternalHelper. return InternalHelper::New(); From a6a238065e61b197b1ac69d61049dff3a7ea4448 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 1 Mar 2023 08:57:09 -0800 Subject: [PATCH 134/463] Fix parsing/serialization of packed proto3 custom options We have an obscure bug where packed-by-default repeated custom options defined in proto3 files are not correctly treated as packed when dynamic extensions are used. This CL introduces a simple fix and adds a test verifying the correct behavior. The bug was not a major problem in most cases, since parsers will accept both the packed and unpacked encodings. It is useful to fix this to ensure that descriptors are serialized in a consistent way, though. PiperOrigin-RevId: 513249881 --- src/google/protobuf/extension_set_heavy.cc | 2 +- src/google/protobuf/extension_set_unittest.cc | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/extension_set_heavy.cc b/src/google/protobuf/extension_set_heavy.cc index cdc4cc34ee27..27b5dd615155 100644 --- a/src/google/protobuf/extension_set_heavy.cc +++ b/src/google/protobuf/extension_set_heavy.cc @@ -285,7 +285,7 @@ bool DescriptorPoolExtensionFinder::Find(int number, ExtensionInfo* output) { } else { output->type = extension->type(); output->is_repeated = extension->is_repeated(); - output->is_packed = extension->options().packed(); + output->is_packed = extension->is_packed(); output->descriptor = extension; if (extension->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { output->message_info.prototype = diff --git a/src/google/protobuf/extension_set_unittest.cc b/src/google/protobuf/extension_set_unittest.cc index ccd1c9b8d5cf..b94440429145 100644 --- a/src/google/protobuf/extension_set_unittest.cc +++ b/src/google/protobuf/extension_set_unittest.cc @@ -1359,6 +1359,54 @@ TEST(ExtensionSetTest, DynamicExtensions) { } } +TEST(ExtensionSetTest, Proto3PackedDynamicExtensions) { + // Regression test for b/271121265. This test case verifies that + // packed-by-default repeated custom options in proto3 are correctly + // serialized in packed form when dynamic extensions are used. + + // Create a custom option in proto3 and load this into an overlay + // DescriptorPool with a DynamicMessageFactory. + google::protobuf::FileDescriptorProto file_descriptor_proto; + file_descriptor_proto.set_syntax("proto3"); + file_descriptor_proto.set_name( + "third_party/protobuf/unittest_proto3_packed_extension.proto"); + file_descriptor_proto.set_package("proto3_unittest"); + file_descriptor_proto.add_dependency( + DescriptorProto::descriptor()->file()->name()); + FieldDescriptorProto* extension = file_descriptor_proto.add_extension(); + extension->set_name("repeated_int32_option"); + extension->set_extendee(MessageOptions().GetTypeName()); + extension->set_number(50009); + extension->set_label(FieldDescriptorProto::LABEL_REPEATED); + extension->set_type(FieldDescriptorProto::TYPE_INT32); + extension->set_json_name("repeatedInt32Option"); + google::protobuf::DescriptorPool pool(DescriptorPool::generated_pool()); + ASSERT_NE(pool.BuildFile(file_descriptor_proto), nullptr); + DynamicMessageFactory factory; + factory.SetDelegateToGeneratedFactory(true); + + // Create a serialized MessageOptions proto equivalent to: + // [proto3_unittest.repeated_int32_option]: 1 + UnknownFieldSet unknown_fields; + unknown_fields.AddVarint(50009, 1); + std::string serialized_options; + ASSERT_TRUE(unknown_fields.SerializeToString(&serialized_options)); + + // Parse the MessageOptions using our custom extension registry. + io::ArrayInputStream input_stream(serialized_options.data(), + serialized_options.size()); + io::CodedInputStream coded_stream(&input_stream); + coded_stream.SetExtensionRegistry(&pool, &factory); + MessageOptions message_options; + ASSERT_TRUE(message_options.ParseFromCodedStream(&coded_stream)); + + // Finally, serialize the proto again and verify that the repeated option has + // been correctly serialized in packed form. + std::string reserialized_options; + ASSERT_TRUE(message_options.SerializeToString(&reserialized_options)); + EXPECT_EQ(reserialized_options, "\xca\xb5\x18\x01\x01"); +} + TEST(ExtensionSetTest, BoolExtension) { unittest::TestAllExtensions msg; uint8_t wire_bytes[2] = {13 * 8, 42 /* out of bounds payload for bool */}; From dc4b09e13879d06618c6a66096e0c6819ff28d28 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 Mar 2023 09:17:14 -0800 Subject: [PATCH 135/463] Add mini-parse support for map fields to table-driven parser. PiperOrigin-RevId: 513255347 --- src/google/protobuf/BUILD.bazel | 2 + .../compiler/cpp/parse_function_generator.cc | 26 +- .../protobuf/generated_message_reflection.cc | 29 +- .../protobuf/generated_message_tctable_decl.h | 113 ++++++ .../protobuf/generated_message_tctable_gen.cc | 61 ++-- .../protobuf/generated_message_tctable_gen.h | 4 + .../protobuf/generated_message_tctable_impl.h | 44 +++ .../generated_message_tctable_lite.cc | 332 +++++++++++++++++- src/google/protobuf/map.h | 82 ++++- src/google/protobuf/map_field.h | 18 +- src/google/protobuf/map_field_lite.h | 2 + src/google/protobuf/map_field_test.cc | 1 + src/google/protobuf/map_proto2_unittest.proto | 33 ++ src/google/protobuf/map_proto3_unittest.proto | 43 +++ src/google/protobuf/map_test.inc | 155 +++++++- src/google/protobuf/parse_context.h | 24 ++ src/google/protobuf/port_def.inc | 15 + src/google/protobuf/port_undef.inc | 1 + 18 files changed, 937 insertions(+), 48 deletions(-) create mode 100644 src/google/protobuf/map_proto3_unittest.proto diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index cb2835b36d3a..34ee53bcd9d8 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -550,6 +550,7 @@ filegroup( srcs = [ "any_test.proto", "map_proto2_unittest.proto", + "map_proto3_unittest.proto", "map_unittest.proto", "unittest.proto", "unittest_arena.proto", @@ -606,6 +607,7 @@ proto_library( name = "generic_test_protos", srcs = [ "map_proto2_unittest.proto", + "map_proto3_unittest.proto", "map_unittest.proto", "unittest.proto", "unittest_arena.proto", diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index a296d411e71c..864ea45b5bdd 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -96,7 +96,8 @@ class ParseFunctionGenerator::GeneratedOptionProvider final UseDirectTcParserTable(field, gen_->options_), GetOptimizeFor(field->file(), gen_->options_) == FileOptions::LITE_RUNTIME, - ShouldSplit(field, gen_->options_)}; + ShouldSplit(field, gen_->options_), + true}; } private: @@ -616,6 +617,29 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { case TailCallTableInfo::kNumericOffset: format("{_fl::Offset{$1$}},\n", aux_entry.offset); break; + case TailCallTableInfo::kMapAuxInfo: { + auto utf8_check = internal::cpp::GetUtf8CheckMode( + aux_entry.field, + GetOptimizeFor(aux_entry.field->file(), options_) == + FileOptions::LITE_RUNTIME); + auto* map_value = aux_entry.field->message_type()->map_value(); + const bool validated_enum = + map_value->type() == FieldDescriptor::TYPE_ENUM && + !internal::cpp::HasPreservingUnknownEnumSemantics( + map_value); + format( + "{::_pbi::TcParser::GetMapAuxInfo($2$, $3$, $4$)},\n", + FieldMemberName(aux_entry.field, /*split=*/false), + utf8_check == internal::cpp::Utf8CheckMode::kStrict, + utf8_check == internal::cpp::Utf8CheckMode::kVerify, + validated_enum); + break; + } + case TailCallTableInfo::kCreateInArena: + format("{::_pbi::TcParser::CreateInArenaStorageCb<$1$>},\n", + QualifiedClassName(aux_entry.desc, options_)); + break; } } } diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index f5c6b22292f4..ed5a868ff71c 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -3089,8 +3089,16 @@ void Reflection::PopulateTcParseFieldAux( break; case internal::TailCallTableInfo::kSubTable: case internal::TailCallTableInfo::kSubMessageWeak: + case internal::TailCallTableInfo::kCreateInArena: ABSL_LOG(FATAL) << "Not supported"; break; + case internal::TailCallTableInfo::kMapAuxInfo: + // Default constructed info, which causes MpMap to call the fallback. + // DynamicMessage uses DynamicMapField, which uses variant keys and + // values. TcParser does not support them yet, so mark the field as + // unsupported to fallback to reflection. + field_aux++->map_info = internal::MapAuxInfo{}; + break; case internal::TailCallTableInfo::kSubMessage: field_aux++->message_default_p = GetDefaultMessageInstance(aux_entry.field); @@ -3143,18 +3151,21 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { explicit ReflectionOptionProvider(const Reflection& ref) : ref_(ref) {} internal::TailCallTableInfo::PerFieldOptions GetForField( const FieldDescriptor* field) const final { - return {ref_.IsLazyField(field), // - ref_.IsInlined(field), // + return { + ref_.IsLazyField(field), // + ref_.IsInlined(field), // - // Only LITE can be implicitly weak. - /* is_implicitly_weak */ false, + // Only LITE can be implicitly weak. + /* is_implicitly_weak */ false, - // We could change this to use direct table. - // Might be easier to do when all messages support TDP. - /* use_direct_tcparser_table */ false, + // We could change this to use direct table. + // Might be easier to do when all messages support TDP. + /* use_direct_tcparser_table */ false, - /* is_lite */ false, // - ref_.schema_.IsSplit(field)}; + /* is_lite */ false, // + ref_.schema_.IsSplit(field), // + /* uses_codegen */ false // + }; } private: diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index 33b094efff19..f9affbe1b33c 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -165,6 +165,114 @@ struct Offset { struct FieldAuxDefaultMessage {}; +// Small type card used by mini parse to handle map entries. +// Map key/values are very limited, so we can encode the whole thing in a single +// byte. +class MapTypeCard { + public: + enum CppType { kBool, k32, k64, kString, kMessage }; + MapTypeCard() = default; + constexpr MapTypeCard(WireFormatLite::WireType wiretype, CppType cpp_type, + bool is_zigzag_utf8) + : data_( + static_cast((static_cast(wiretype) << 0) | + (static_cast(cpp_type) << 3) | + (static_cast(is_zigzag_utf8) << 6))) { + } + + WireFormatLite::WireType wiretype() const { + return static_cast((data_ >> 0) & 0x7); + } + + CppType cpp_type() const { return static_cast((data_ >> 3) & 0x7); } + + bool is_zigzag() const { + ABSL_DCHECK(wiretype() == WireFormatLite::WIRETYPE_VARINT); + ABSL_DCHECK(cpp_type() == CppType::k32 || cpp_type() == CppType::k64); + return is_zigzag_utf8(); + } + bool is_utf8() const { + ABSL_DCHECK(wiretype() == WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + ABSL_DCHECK(cpp_type() == CppType::kString); + return is_zigzag_utf8(); + } + + private: + bool is_zigzag_utf8() const { return static_cast((data_ >> 6) & 0x1); } + uint8_t data_; +}; +static_assert(sizeof(MapTypeCard) == sizeof(uint8_t), ""); + +// Make the map entry type card for a specified field type. +constexpr MapTypeCard MakeMapTypeCard(WireFormatLite::FieldType type) { + switch (type) { + case WireFormatLite::TYPE_FLOAT: + case WireFormatLite::TYPE_FIXED32: + case WireFormatLite::TYPE_SFIXED32: + return {WireFormatLite::WIRETYPE_FIXED32, MapTypeCard::k32, false}; + + case WireFormatLite::TYPE_DOUBLE: + case WireFormatLite::TYPE_FIXED64: + case WireFormatLite::TYPE_SFIXED64: + return {WireFormatLite::WIRETYPE_FIXED64, MapTypeCard::k64, false}; + + case WireFormatLite::TYPE_BOOL: + return {WireFormatLite::WIRETYPE_VARINT, MapTypeCard::kBool, false}; + + case WireFormatLite::TYPE_ENUM: + // Enum validation is handled via `value_is_validated_enum` below. + case WireFormatLite::TYPE_INT32: + case WireFormatLite::TYPE_UINT32: + return {WireFormatLite::WIRETYPE_VARINT, MapTypeCard::k32, false}; + + case WireFormatLite::TYPE_INT64: + case WireFormatLite::TYPE_UINT64: + return {WireFormatLite::WIRETYPE_VARINT, MapTypeCard::k64, false}; + + case WireFormatLite::TYPE_SINT32: + return {WireFormatLite::WIRETYPE_VARINT, MapTypeCard::k32, true}; + case WireFormatLite::TYPE_SINT64: + return {WireFormatLite::WIRETYPE_VARINT, MapTypeCard::k64, true}; + + case WireFormatLite::TYPE_STRING: + return {WireFormatLite::WIRETYPE_LENGTH_DELIMITED, MapTypeCard::kString, + true}; + case WireFormatLite::TYPE_BYTES: + return {WireFormatLite::WIRETYPE_LENGTH_DELIMITED, MapTypeCard::kString, + false}; + + case WireFormatLite::TYPE_MESSAGE: + return {WireFormatLite::WIRETYPE_LENGTH_DELIMITED, MapTypeCard::kMessage, + false}; + + default: + PROTOBUF_ASSUME(false); + } +} + +enum class MapNodeSizeInfoT : uint32_t; + +// Aux entry for map fields. +struct MapAuxInfo { + MapTypeCard key_type_card; + MapTypeCard value_type_card; + // When off, we fall back to table->fallback to handle the parse. An example + // of this is for DynamicMessage. + uint8_t is_supported : 1; + // Determines if we are using LITE or the full runtime. When using the full + // runtime we have to synchronize with reflection before accessing the map. + uint8_t use_lite : 1; + // If true UTF8 errors cause the parsing to fail. + uint8_t fail_on_utf8_failure : 1; + // If true UTF8 errors are logged, but they are accepted. + uint8_t log_debug_utf8_failure : 1; + // If true the next aux contains the enum validator. + uint8_t value_is_validated_enum : 1; + // Size information derived from the actual node type. + MapNodeSizeInfoT node_size_info; +}; +static_assert(sizeof(MapAuxInfo) <= 8, ""); + // Base class for message-level table with info for the tail-call parser. struct alignas(uint64_t) TcParseTableBase { // Common attributes for message layout: @@ -291,6 +399,9 @@ struct alignas(uint64_t) TcParseTableBase { constexpr FieldAux(FieldAuxDefaultMessage, const void* msg) : message_default_p(msg) {} constexpr FieldAux(const TcParseTableBase* table) : table(table) {} + constexpr FieldAux(MapAuxInfo map_info) : map_info(map_info) {} + constexpr FieldAux(void (*create_in_arena)(Arena*, void*)) + : create_in_arena(create_in_arena) {} bool (*enum_validator)(int); struct { int16_t start; // minimum enum number (if it fits) @@ -299,6 +410,8 @@ struct alignas(uint64_t) TcParseTableBase { uint32_t offset; const void* message_default_p; const TcParseTableBase* table; + MapAuxInfo map_info; + void (*create_in_arena)(Arena*, void*); const MessageLite* message_default() const { return static_cast(message_default_p); diff --git a/src/google/protobuf/generated_message_tctable_gen.cc b/src/google/protobuf/generated_message_tctable_gen.cc index c5f8e195b191..987b669336f1 100644 --- a/src/google/protobuf/generated_message_tctable_gen.cc +++ b/src/google/protobuf/generated_message_tctable_gen.cc @@ -425,7 +425,7 @@ std::vector FilterMiniParsedFields( case FieldDescriptor::TYPE_MESSAGE: case FieldDescriptor::TYPE_GROUP: // TODO(b/210762816): support remaining field types. - if (field->is_map() || field->options().weak() || options.is_lazy) { + if (field->options().weak() || options.is_lazy) { handled = false; } else { handled = true; @@ -444,33 +444,38 @@ std::vector FilterMiniParsedFields( // We only need field names for reporting UTF-8 parsing errors, so we only // emit them for string fields with Utf8 transform specified. -absl::string_view FieldNameForTable( - const TailCallTableInfo::FieldEntryInfo& entry) { - const auto* field = entry.field; - if (field->type() == FieldDescriptor::TYPE_STRING) { - const uint16_t xform_val = entry.type_card & field_layout::kTvMask; +bool NeedsFieldNameForTable(const FieldDescriptor* field, bool is_lite) { + if (cpp::GetUtf8CheckMode(field, is_lite) == cpp::Utf8CheckMode::kNone) + return false; + return field->type() == FieldDescriptor::TYPE_STRING || + (field->is_map() && (field->message_type()->map_key()->type() == + FieldDescriptor::TYPE_STRING || + field->message_type()->map_value()->type() == + FieldDescriptor::TYPE_STRING)); +} - switch (xform_val) { - case field_layout::kTvUtf8: - case field_layout::kTvUtf8Debug: - return field->name(); - } +absl::string_view FieldNameForTable( + const TailCallTableInfo::FieldEntryInfo& entry, + const TailCallTableInfo::OptionProvider& option_provider) { + if (NeedsFieldNameForTable( + entry.field, option_provider.GetForField(entry.field).is_lite)) { + return entry.field->name(); } return ""; } std::vector GenerateFieldNames( const Descriptor* descriptor, - const std::vector& entries) { + const std::vector& entries, + const TailCallTableInfo::OptionProvider& option_provider) { static constexpr int kMaxNameLength = 255; std::vector out; + std::vector names; bool found_needed_name = false; for (const auto& entry : entries) { - if (!FieldNameForTable(entry).empty()) { - found_needed_name = true; - break; - } + names.push_back(FieldNameForTable(entry, option_provider)); + if (!names.back().empty()) found_needed_name = true; } // No names needed. Omit the whole table. @@ -483,8 +488,8 @@ std::vector GenerateFieldNames( int count = 1; out.push_back(std::min(static_cast(descriptor->full_name().size()), kMaxNameLength)); - for (const auto& entry : entries) { - out.push_back(FieldNameForTable(entry).size()); + for (auto field_name : names) { + out.push_back(field_name.size()); ++count; } while (count & 7) { // align to an 8-byte boundary @@ -501,8 +506,7 @@ std::vector GenerateFieldNames( } out.insert(out.end(), message_name.begin(), message_name.end()); // Then we output the actual field names - for (const auto& entry : entries) { - const auto& field_name = FieldNameForTable(entry); + for (auto field_name : names) { out.insert(out.end(), field_name.begin(), field_name.end()); } @@ -779,7 +783,19 @@ TailCallTableInfo::TailCallTableInfo( field->type() == FieldDescriptor::TYPE_GROUP) { // Message-typed fields have a FieldAux with the default instance pointer. if (field->is_map()) { - // TODO(b/205904770): generate aux entries for maps + field_entries.back().aux_idx = aux_entries.size(); + aux_entries.push_back({kMapAuxInfo, {field}}); + if (options.uses_codegen) { + // If we don't use codegen we can't add these. + auto* map_value = field->message_type()->map_value(); + if (auto* sub = map_value->message_type()) { + aux_entries.push_back({kCreateInArena}); + aux_entries.back().desc = sub; + } else if (map_value->type() == FieldDescriptor::TYPE_ENUM && + !cpp::HasPreservingUnknownEnumSemantics(map_value)) { + aux_entries.push_back({kEnumValidator, {map_value}}); + } + } } else if (field->options().weak()) { // Don't generate anything for weak fields. They are handled by the // generated fallback. @@ -880,7 +896,8 @@ TailCallTableInfo::TailCallTableInfo( num_to_entry_table = MakeNumToEntryTable(ordered_fields); ABSL_CHECK_EQ(field_entries.size(), ordered_fields.size()); - field_name_data = GenerateFieldNames(descriptor, field_entries); + field_name_data = + GenerateFieldNames(descriptor, field_entries, option_provider); // If there are no fallback fields, and at most one extension range, the // parser can use a generic fallback function. Otherwise, a message-specific diff --git a/src/google/protobuf/generated_message_tctable_gen.h b/src/google/protobuf/generated_message_tctable_gen.h index ac24dd7766ef..3f5f7ec13a82 100644 --- a/src/google/protobuf/generated_message_tctable_gen.h +++ b/src/google/protobuf/generated_message_tctable_gen.h @@ -59,6 +59,7 @@ struct PROTOBUF_EXPORT TailCallTableInfo { bool use_direct_tcparser_table; bool is_lite; bool should_split; + bool uses_codegen; }; class OptionProvider { public: @@ -106,6 +107,8 @@ struct PROTOBUF_EXPORT TailCallTableInfo { kEnumRange, kEnumValidator, kNumericOffset, + kMapAuxInfo, + kCreateInArena, }; struct AuxEntry { AuxType type; @@ -115,6 +118,7 @@ struct PROTOBUF_EXPORT TailCallTableInfo { }; union { const FieldDescriptor* field; + const Descriptor* desc; uint32_t offset; EnumRange enum_range; }; diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 1975639c4c3f..547459b84a34 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -40,6 +40,7 @@ #include "google/protobuf/port.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/generated_message_tctable_decl.h" +#include "google/protobuf/map.h" #include "google/protobuf/metadata_lite.h" #include "google/protobuf/parse_context.h" #include "google/protobuf/wire_format_lite.h" @@ -617,6 +618,33 @@ class PROTOBUF_EXPORT TcParser final { static const char* FastEndG1(PROTOBUF_TC_PARAM_DECL); static const char* FastEndG2(PROTOBUF_TC_PARAM_DECL); + // For `map` mini parsing generate a type card for the key/value. + template + static constexpr MapAuxInfo GetMapAuxInfo(bool fail_on_utf8_failure, + bool log_debug_utf8_failure, + bool validated_enum_value) { + using MapType = typename MapField::MapType; + using Node = typename MapType::Node; + static_assert(alignof(Node) == alignof(NodeBase), ""); + // Verify the assumption made in MpMap, guaranteed by Map<>. + assert(PROTOBUF_FIELD_OFFSET(Node, kv.first) == sizeof(NodeBase)); + return { + MakeMapTypeCard(MapField::kKeyFieldType), + MakeMapTypeCard(MapField::kValueFieldType), + true, + !std::is_base_of::value, + fail_on_utf8_failure, + log_debug_utf8_failure, + validated_enum_value, + Node::size_info(), + }; + } + + template + static void CreateInArenaStorageCb(Arena* arena, void* p) { + Arena::CreateInArenaStorage(static_cast(p), arena); + } + private: // Optimized small tag varint parser for int32/int64 template @@ -774,6 +802,21 @@ class PROTOBUF_EXPORT TcParser final { static void AddUnknownEnum(MessageLite* msg, const TcParseTableBase* table, uint32_t tag, int32_t enum_value); + static void WriteMapEntryAsUnknown(MessageLite* msg, + const TcParseTableBase* table, + uint32_t tag, NodeBase* node, + MapAuxInfo map_info); + + static void InitializeMapNodeEntry(void* obj, MapTypeCard type_card, + UntypedMapBase& map, + const TcParseTableBase::FieldAux* aux); + static void DestroyMapNode(NodeBase* node, MapAuxInfo map_info, + UntypedMapBase& map); + static const char* ParseOneMapEntry( + NodeBase* node, const char* ptr, ParseContext* ctx, + const TcParseTableBase::FieldAux* aux, const TcParseTableBase* table, + const TcParseTableBase::FieldEntry& entry); + // Mini field lookup: static const TcParseTableBase::FieldEntry* FindFieldEntry( const TcParseTableBase* table, uint32_t field_num); @@ -815,6 +858,7 @@ class PROTOBUF_EXPORT TcParser final { static const char* MpMessage(PROTOBUF_TC_PARAM_DECL); static const char* MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL); static const char* MpFallback(PROTOBUF_TC_PARAM_DECL); + static const char* MpMap(PROTOBUF_TC_PARAM_DECL); }; // Dispatch to the designated parse function diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 64d955c1b73b..95869ff11bca 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -37,6 +37,8 @@ #include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/inlined_string_field.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/map.h" #include "google/protobuf/message_lite.h" #include "google/protobuf/parse_context.h" #include "google/protobuf/wire_format_lite.h" @@ -308,7 +310,7 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::MiniParse( &MpPackedFixed, // FieldKind::kFkPackedFixed &MpString, // FieldKind::kFkString &MpMessage, // FieldKind::kFkMessage - &MpFallback, // FieldKind::kFkMap + &MpMap, // FieldKind::kFkMap &Error, // kSplitMask | FieldKind::kFkNone &MpVarint, // kSplitMask | FieldKind::kFkVarint &Error, // kSplitMask | FieldKind::kFkPackedVarint @@ -363,7 +365,7 @@ PROTOBUF_NOINLINE TcParser::TestMiniParseResult TcParser::TestMiniParse( return result; } -const char* TcParser::MpFallback(PROTOBUF_TC_PARAM_DECL) { +PROTOBUF_NOINLINE const char* TcParser::MpFallback(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); } @@ -2492,6 +2494,332 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { } } +static void SerializeMapKey(const NodeBase* node, MapTypeCard type_card, + io::CodedOutputStream& coded_output) { + switch (type_card.wiretype()) { + case WireFormatLite::WIRETYPE_VARINT: + switch (type_card.cpp_type()) { + case MapTypeCard::kBool: + WireFormatLite::WriteBool( + 1, static_cast*>(node)->key(), &coded_output); + break; + case MapTypeCard::k32: + if (type_card.is_zigzag()) { + WireFormatLite::WriteSInt32( + 1, static_cast*>(node)->key(), + &coded_output); + } else { + WireFormatLite::WriteInt32( + 1, static_cast*>(node)->key(), + &coded_output); + } + break; + case MapTypeCard::k64: + if (type_card.is_zigzag()) { + WireFormatLite::WriteSInt64( + 1, static_cast*>(node)->key(), + &coded_output); + } else { + WireFormatLite::WriteInt64( + 1, static_cast*>(node)->key(), + &coded_output); + } + break; + default: + PROTOBUF_ASSUME(false); + } + break; + case WireFormatLite::WIRETYPE_FIXED32: + WireFormatLite::WriteFixed32( + 1, static_cast*>(node)->key(), &coded_output); + break; + case WireFormatLite::WIRETYPE_FIXED64: + WireFormatLite::WriteFixed64( + 1, static_cast*>(node)->key(), &coded_output); + break; + case WireFormatLite::WIRETYPE_LENGTH_DELIMITED: + // We should never have a message here. They can only be values maps. + ABSL_DCHECK_EQ(+type_card.cpp_type(), +MapTypeCard::kString); + WireFormatLite::WriteString( + 1, static_cast*>(node)->key(), + &coded_output); + break; + default: + PROTOBUF_ASSUME(false); + } +} + +void TcParser::WriteMapEntryAsUnknown(MessageLite* msg, + const TcParseTableBase* table, + uint32_t tag, NodeBase* node, + MapAuxInfo map_info) { + std::string serialized; + { + io::StringOutputStream string_output(&serialized); + io::CodedOutputStream coded_output(&string_output); + SerializeMapKey(node, map_info.key_type_card, coded_output); + // The mapped_type is always an enum here. + ABSL_DCHECK(map_info.value_is_validated_enum); + WireFormatLite::WriteInt32(2, + *reinterpret_cast( + node->GetVoidValue(map_info.node_size_info)), + &coded_output); + } + GetUnknownFieldOps(table).write_length_delimited(msg, tag >> 3, serialized); +} + +PROTOBUF_ALWAYS_INLINE inline void TcParser::InitializeMapNodeEntry( + void* obj, MapTypeCard type_card, UntypedMapBase& map, + const TcParseTableBase::FieldAux* aux) { + switch (type_card.cpp_type()) { + case MapTypeCard::kBool: + memset(obj, 0, sizeof(bool)); + break; + case MapTypeCard::k32: + memset(obj, 0, sizeof(uint32_t)); + break; + case MapTypeCard::k64: + memset(obj, 0, sizeof(uint64_t)); + break; + case MapTypeCard::kString: + Arena::CreateInArenaStorage(reinterpret_cast(obj), + map.arena()); + break; + case MapTypeCard::kMessage: + aux[1].create_in_arena(map.arena(), reinterpret_cast(obj)); + break; + default: + PROTOBUF_ASSUME(false); + } +} + +PROTOBUF_NOINLINE void TcParser::DestroyMapNode(NodeBase* node, + MapAuxInfo map_info, + UntypedMapBase& map) { + if (map_info.key_type_card.cpp_type() == MapTypeCard::kString) { + static_cast(node->GetVoidKey())->~basic_string(); + } + if (map_info.value_type_card.cpp_type() == MapTypeCard::kString) { + static_cast(node->GetVoidValue(map_info.node_size_info)) + ->~basic_string(); + } else if (map_info.value_type_card.cpp_type() == MapTypeCard::kMessage) { + static_cast(node->GetVoidValue(map_info.node_size_info)) + ->~MessageLite(); + } + map.DeallocNode(node, map_info.node_size_info); +} + +template +const char* ReadFixed(void* obj, const char* ptr) { + auto v = UnalignedLoad(ptr); + ptr += sizeof(v); + memcpy(obj, &v, sizeof(v)); + return ptr; +} + +const char* TcParser::ParseOneMapEntry( + NodeBase* node, const char* ptr, ParseContext* ctx, + const TcParseTableBase::FieldAux* aux, const TcParseTableBase* table, + const TcParseTableBase::FieldEntry& entry) { + using WFL = WireFormatLite; + + const auto map_info = aux[0].map_info; + const uint8_t key_tag = WFL::MakeTag(1, map_info.key_type_card.wiretype()); + const uint8_t value_tag = + WFL::MakeTag(2, map_info.value_type_card.wiretype()); + + while (!ctx->Done(&ptr)) { + uint32_t inner_tag = ptr[0]; + + if (PROTOBUF_PREDICT_FALSE(inner_tag != key_tag && + inner_tag != value_tag)) { + // Do a full parse and check again in case the tag has non-canonical + // encoding. + ptr = ReadTag(ptr, &inner_tag); + if (PROTOBUF_PREDICT_FALSE(inner_tag != key_tag && + inner_tag != value_tag)) { + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + + if (inner_tag == 0 || (inner_tag & 7) == WFL::WIRETYPE_END_GROUP) { + ctx->SetLastTag(inner_tag); + break; + } + + ptr = UnknownFieldParse(inner_tag, nullptr, ptr, ctx); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + continue; + } + } else { + ++ptr; + } + + MapTypeCard type_card; + void* obj; + if (inner_tag == key_tag) { + type_card = map_info.key_type_card; + obj = node->GetVoidKey(); + } else { + type_card = map_info.value_type_card; + obj = node->GetVoidValue(map_info.node_size_info); + } + + switch (type_card.wiretype()) { + case WFL::WIRETYPE_VARINT: + uint64_t tmp; + ptr = ParseVarint(ptr, &tmp); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + switch (type_card.cpp_type()) { + case MapTypeCard::kBool: + *reinterpret_cast(obj) = static_cast(tmp); + continue; + case MapTypeCard::k32: { + uint32_t v = static_cast(tmp); + if (type_card.is_zigzag()) v = WFL::ZigZagDecode32(v); + memcpy(obj, &v, sizeof(v)); + continue; + } + case MapTypeCard::k64: + if (type_card.is_zigzag()) tmp = WFL::ZigZagDecode64(tmp); + memcpy(obj, &tmp, sizeof(tmp)); + continue; + default: + PROTOBUF_ASSUME(false); + } + case WFL::WIRETYPE_FIXED32: + ptr = ReadFixed(obj, ptr); + continue; + case WFL::WIRETYPE_FIXED64: + ptr = ReadFixed(obj, ptr); + continue; + case WFL::WIRETYPE_LENGTH_DELIMITED: + if (type_card.cpp_type() == MapTypeCard::kString) { + const int size = ReadSize(&ptr); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + std::string* str = reinterpret_cast(obj); + ptr = ctx->ReadString(ptr, size, str); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + bool do_utf8_check = map_info.fail_on_utf8_failure; +#ifndef NDEBUG + do_utf8_check |= map_info.log_debug_utf8_failure; +#endif + if (type_card.is_utf8() && do_utf8_check && + !utf8_range::IsStructurallyValid(*str)) { + PrintUTF8ErrorLog(MessageName(table), FieldName(table, &entry), + "parsing", false); + if (map_info.fail_on_utf8_failure) { + return nullptr; + } + } + continue; + } else { + ABSL_DCHECK_EQ(+type_card.cpp_type(), +MapTypeCard::kMessage); + ABSL_DCHECK_EQ(inner_tag, value_tag); + ptr = ctx->ParseMessage(reinterpret_cast(obj), ptr); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + continue; + } + default: + PROTOBUF_ASSUME(false); + } + } + return ptr; +} + +PROTOBUF_NOINLINE const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) { + const auto& entry = RefAt(table, data.entry_offset()); + // `aux[0]` points into a MapAuxInfo. + // If we have a message mapped_type aux[1] points into a `create_in_arena`. + // If we have a validated enum mapped_type aux[1] point into a + // `enum_validator`. + const auto* aux = table->field_aux(&entry); + const auto map_info = aux[0].map_info; + + if (PROTOBUF_PREDICT_FALSE(!map_info.is_supported || + (data.tag() & 7) != + WireFormatLite::WIRETYPE_LENGTH_DELIMITED)) { + PROTOBUF_MUSTTAIL return MpFallback(PROTOBUF_TC_PARAM_PASS); + } + + // When using LITE, the offset points directly into the Map<> object. + // Otherwise, it points into a MapField and we must synchronize with + // reflection. It is done by calling the MutableMap() virtual function on the + // field's base class. + UntypedMapBase& map = + map_info.use_lite + ? RefAt(msg, entry.offset) + : *RefAt(msg, entry.offset).MutableMap(); + + const uint32_t saved_tag = data.tag(); + + while (true) { + NodeBase* node = map.AllocNode(map_info.node_size_info); + + InitializeMapNodeEntry(node->GetVoidKey(), map_info.key_type_card, map, + aux); + InitializeMapNodeEntry(node->GetVoidValue(map_info.node_size_info), + map_info.value_type_card, map, aux); + + ptr = ctx->ParseLengthDelimitedInlined(ptr, [&](const char* ptr) { + return ParseOneMapEntry(node, ptr, ctx, aux, table, entry); + }); + + if (PROTOBUF_PREDICT_TRUE(ptr != nullptr)) { + if (PROTOBUF_PREDICT_FALSE( + map_info.value_is_validated_enum && + !aux[1].enum_validator(*static_cast( + node->GetVoidValue(map_info.node_size_info))))) { + WriteMapEntryAsUnknown(msg, table, saved_tag, node, map_info); + } else { + // Done parsing the node, try to insert it. + // If it overwrites something we get old node back to destroy it. + switch (map_info.key_type_card.cpp_type()) { + case MapTypeCard::kBool: + node = static_cast&>(map).InsertOrReplaceNode( + static_cast::KeyNode*>(node)); + break; + case MapTypeCard::k32: + node = static_cast&>(map).InsertOrReplaceNode( + static_cast::KeyNode*>(node)); + break; + case MapTypeCard::k64: + node = static_cast&>(map).InsertOrReplaceNode( + static_cast::KeyNode*>(node)); + break; + case MapTypeCard::kString: + node = + static_cast&>(map).InsertOrReplaceNode( + static_cast::KeyNode*>(node)); + break; + default: + PROTOBUF_ASSUME(false); + } + } + } + + // Destroy the node if we have it. + // It could be because we failed to parse, or because insertion returned + // an overwritten node. + if (PROTOBUF_PREDICT_FALSE(node != nullptr && map.arena() == nullptr)) { + DestroyMapNode(node, map_info, map); + } + + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); + } + + uint32_t next_tag; + const char* ptr2 = ReadTagInlined(ptr, &next_tag); + if (next_tag != saved_tag) break; + ptr = ptr2; + } + + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index e3bc9ecc7c9a..99369d3b54e6 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -90,6 +90,8 @@ template class MapField; +struct MapTestPeer; + template class TypeDefinedMapFieldBase; @@ -291,11 +293,30 @@ struct TransparentSupport { using key_arg = K; }; +enum class MapNodeSizeInfoT : uint32_t; +inline uint16_t SizeFromInfo(MapNodeSizeInfoT node_size_info) { + return static_cast(static_cast(node_size_info) >> 16); +} +inline uint16_t ValueOffsetFromInfo(MapNodeSizeInfoT node_size_info) { + return static_cast(static_cast(node_size_info) >> 0); +} +constexpr MapNodeSizeInfoT MakeNodeInfo(uint16_t size, uint16_t value_offset) { + return static_cast((static_cast(size) << 16) | + value_offset); +} + struct NodeBase { // Align the node to allow KeyNode to predict the location of the key. // This way sizeof(NodeBase) contains any possible padding it was going to // have between NodeBase and the key. alignas(kMaxMessageAlignment) NodeBase* next; + + void* GetVoidKey() { return this + 1; } + const void* GetVoidKey() const { return this + 1; } + + void* GetVoidValue(MapNodeSizeInfoT size_info) { + return reinterpret_cast(this) + ValueOffsetFromInfo(size_info); + } }; inline NodeBase* EraseFromLinkedList(NodeBase* item, NodeBase* head) { @@ -474,6 +495,7 @@ class PROTOBUF_EXPORT UntypedMapBase { protected: friend class TcParser; + friend struct MapTestPeer; struct NodeAndBucket { NodeBase* node; @@ -538,11 +560,19 @@ class PROTOBUF_EXPORT UntypedMapBase { using AllocFor = absl::allocator_traits::template rebind_alloc; // Alignment of the nodes is the same as alignment of NodeBase. + NodeBase* AllocNode(MapNodeSizeInfoT size_info) { + return AllocNode(SizeFromInfo(size_info)); + } + NodeBase* AllocNode(size_t node_size) { PROTOBUF_ASSUME(node_size % sizeof(NodeBase) == 0); return AllocFor(alloc_).allocate(node_size / sizeof(NodeBase)); } + void DeallocNode(NodeBase* node, MapNodeSizeInfoT size_info) { + DeallocNode(node, SizeFromInfo(size_info)); + } + void DeallocNode(NodeBase* node, size_t node_size) { PROTOBUF_ASSUME(node_size % sizeof(NodeBase) == 0); AllocFor(alloc_).deallocate(node, node_size / sizeof(NodeBase)); @@ -595,6 +625,17 @@ class PROTOBUF_EXPORT UntypedMapBase { Allocator alloc_; }; +// Base class used by TcParser to extract the map object from a map field. +// We keep it here to avoid a dependency into map_field.h from the main TcParser +// code, since that would bring in Message too. +class MapFieldBaseForParse { + public: + virtual UntypedMapBase* MutableMap() = 0; + + protected: + ~MapFieldBaseForParse() = default; +}; + // The value might be of different signedness, so use memcpy to extract it. template ::value, int> = 0> T ReadKey(const void* ptr) { @@ -608,6 +649,12 @@ const T& ReadKey(const void* ptr) { return *reinterpret_cast(ptr); } +template +struct KeyNode : NodeBase { + static constexpr size_t kOffset = sizeof(NodeBase); + decltype(auto) key() const { return ReadKey(GetVoidKey()); } +}; + // KeyMapBase is a chaining hash map with the additional feature that some // buckets can be converted to use an ordered container. This ensures O(lg n) // bounds on find, insert, and erase, while avoiding the overheads of ordered @@ -644,12 +691,7 @@ class KeyMapBase : public UntypedMapBase { using UntypedMapBase::UntypedMapBase; protected: - struct KeyNode : NodeBase { - static constexpr size_t kOffset = sizeof(NodeBase); - decltype(auto) key() const { - return ReadKey(reinterpret_cast(this) + kOffset); - } - }; + using KeyNode = internal::KeyNode; // Trees. The payload type is a copy of Key, so that we can query the tree // with Keys that are not in any particular data structure. @@ -728,6 +770,9 @@ class KeyMapBase : public UntypedMapBase { hasher hash_function() const { return {}; } protected: + friend class TcParser; + friend struct MapTestPeer; + PROTOBUF_NOINLINE void erase_no_destroy(size_type b, KeyNode* node) { TreeIterator tree_it; const bool is_list = revalidate_if_necessary(b, node, &tree_it); @@ -785,6 +830,25 @@ class KeyMapBase : public UntypedMapBase { return {nullptr, b}; } + // Insert the given node. + // If the key is a duplicate, it inserts the new node and returns the old one. + // Gives ownership to the caller. + // If the key is unique, it returns `nullptr`. + KeyNode* InsertOrReplaceNode(KeyNode* node) { + KeyNode* to_erase = nullptr; + auto p = this->FindHelper(node->key()); + if (p.node != nullptr) { + erase_no_destroy(p.bucket, static_cast(p.node)); + to_erase = static_cast(p.node); + } else if (ResizeIfLoadIsOutOfRange(num_elements_ + 1)) { + p = FindHelper(node->key()); + } + const size_type b = p.bucket; // bucket number + InsertUnique(b, node); + ++num_elements_; + return to_erase; + } + // Insert the given Node in bucket b. If that would make bucket b too big, // and bucket b is not a tree, create a tree for buckets b. // Requires count(*KeyPtrFromNodePtr(node)) == 0 and that b is the correct @@ -1417,6 +1481,10 @@ class Map : private internal::KeyMapBase> { // Linked-list nodes, as one would expect for a chaining hash table. struct Node : Base::KeyNode { + static constexpr internal::MapNodeSizeInfoT size_info() { + return internal::MakeNodeInfo(sizeof(Node), + PROTOBUF_FIELD_OFFSET(Node, kv.second)); + } value_type kv; }; @@ -1543,6 +1611,8 @@ class Map : private internal::KeyMapBase> { internal::WireFormatLite::FieldType key_wire_type, internal::WireFormatLite::FieldType value_wire_type> friend class internal::MapFieldLite; + friend class internal::TcParser; + friend struct internal::MapTestPeer; }; namespace internal { diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index 4529b0e52b24..a6f5f905ddb2 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -336,7 +336,7 @@ class MapFieldAccessor; // This class provides access to map field using reflection, which is the same // as those provided for RepeatedPtrField. It is used for internal // reflection implementation only. Users should never use this directly. -class PROTOBUF_EXPORT MapFieldBase { +class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { public: MapFieldBase() : arena_(nullptr), repeated_field_(nullptr), state_(STATE_MODIFIED_MAP) {} @@ -523,7 +523,9 @@ class TypeDefinedMapFieldBase : public MapFieldBase { bool EqualIterator(const MapIterator& a, const MapIterator& b) const override; virtual const Map& GetMap() const = 0; - virtual Map* MutableMap() = 0; + // This overrides the base's method to specialize the signature via + // covariance, but we do not yet provide an implementation here, so `= 0`. + Map* MutableMap() override = 0; protected: typename Map::const_iterator& InternalGetIterator( @@ -543,19 +545,19 @@ class TypeDefinedMapFieldBase : public MapFieldBase { // internal generated message implementation only. Users should never use this // directly. template + WireFormatLite::FieldType kKeyFieldType_, + WireFormatLite::FieldType kValueFieldType_> class MapField : public TypeDefinedMapFieldBase { // Provide utilities to parse/serialize key/value. Provide utilities to // manipulate internal stored type. - typedef MapTypeHandler KeyTypeHandler; - typedef MapTypeHandler ValueTypeHandler; + typedef MapTypeHandler KeyTypeHandler; + typedef MapTypeHandler ValueTypeHandler; // Define message type for internal repeated field. typedef Derived EntryType; // Define abbreviation for parent MapFieldLite - typedef MapFieldLite + typedef MapFieldLite MapFieldLiteType; // Enum needs to be handled differently from other types because it has @@ -567,6 +569,8 @@ class MapField : public TypeDefinedMapFieldBase { public: typedef Map MapType; + static constexpr WireFormatLite::FieldType kKeyFieldType = kKeyFieldType_; + static constexpr WireFormatLite::FieldType kValueFieldType = kValueFieldType_; MapField() : impl_() {} MapField(const MapField&) = delete; diff --git a/src/google/protobuf/map_field_lite.h b/src/google/protobuf/map_field_lite.h index dd41c183e724..9ecbf8b8f303 100644 --- a/src/google/protobuf/map_field_lite.h +++ b/src/google/protobuf/map_field_lite.h @@ -67,6 +67,8 @@ class MapFieldLite { public: typedef Map MapType; + static constexpr WireFormatLite::FieldType kKeyFieldType = key_wire_type; + static constexpr WireFormatLite::FieldType kValueFieldType = value_wire_type; constexpr MapFieldLite() : map_() {} explicit MapFieldLite(Arena* arena) : map_(arena) {} diff --git a/src/google/protobuf/map_field_test.cc b/src/google/protobuf/map_field_test.cc index f988d6bdca5f..f7e5b32cf760 100644 --- a/src/google/protobuf/map_field_test.cc +++ b/src/google/protobuf/map_field_test.cc @@ -76,6 +76,7 @@ class MapFieldBaseStub : public MapFieldBase { void SetRepeatedDirty() { state_.store(STATE_MODIFIED_REPEATED, std::memory_order_relaxed); } + UntypedMapBase* MutableMap() override { return nullptr; } bool ContainsMapKey(const MapKey& map_key) const override { return false; } bool InsertOrLookupMapValue(const MapKey& map_key, MapValueRef* val) override { diff --git a/src/google/protobuf/map_proto2_unittest.proto b/src/google/protobuf/map_proto2_unittest.proto index 9cda195d8053..ade970d7931c 100644 --- a/src/google/protobuf/map_proto2_unittest.proto +++ b/src/google/protobuf/map_proto2_unittest.proto @@ -55,11 +55,39 @@ enum Proto2MapEnumPlusExtra { message TestEnumMap { map known_map_field = 101; map unknown_map_field = 102; + + // Other maps with all key types to test the unknown entry serialization + map unknown_map_field_int64 = 200; + map unknown_map_field_uint64 = 201; + map unknown_map_field_int32 = 202; + map unknown_map_field_uint32 = 203; + map unknown_map_field_fixed32 = 204; + map unknown_map_field_fixed64 = 205; + map unknown_map_field_bool = 206; + map unknown_map_field_string = 207; + map unknown_map_field_sint32 = 208; + map unknown_map_field_sint64 = 209; + map unknown_map_field_sfixed32 = 210; + map unknown_map_field_sfixed64 = 211; } message TestEnumMapPlusExtra { map known_map_field = 101; map unknown_map_field = 102; + + // Other maps with all key types to test the unknown entry serialization + map unknown_map_field_int64 = 200; + map unknown_map_field_uint64 = 201; + map unknown_map_field_int32 = 202; + map unknown_map_field_uint32 = 203; + map unknown_map_field_fixed32 = 204; + map unknown_map_field_fixed64 = 205; + map unknown_map_field_bool = 206; + map unknown_map_field_string = 207; + map unknown_map_field_sint32 = 208; + map unknown_map_field_sint64 = 209; + map unknown_map_field_sfixed32 = 210; + map unknown_map_field_sfixed64 = 211; } message TestImportEnumMap { @@ -90,3 +118,8 @@ message TestMaps { message TestSubmessageMaps { optional TestMaps m = 1; } + +message TestProto2BytesMap { + map map_bytes = 1; + map map_string = 2; +} diff --git a/src/google/protobuf/map_proto3_unittest.proto b/src/google/protobuf/map_proto3_unittest.proto new file mode 100644 index 000000000000..cea8ef6e8b6f --- /dev/null +++ b/src/google/protobuf/map_proto3_unittest.proto @@ -0,0 +1,43 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +// This file contains definitions that have different behavior in proto3. + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package proto3_unittest; + +message TestProto3BytesMap { + map map_bytes = 1; + map map_string = 2; +} diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 76c5ce864884..8cd3cc2cc15b 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -49,6 +49,7 @@ #include "google/protobuf/testing/googletest.h" #include #include "absl/base/casts.h" +#include "absl/cleanup/cleanup.h" #include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" @@ -64,6 +65,8 @@ #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/map.h" #include "google/protobuf/map_field_inl.h" +#include "google/protobuf/map_proto2_unittest.pb.h" +#include "google/protobuf/map_proto3_unittest.pb.h" #include "google/protobuf/message.h" #include "google/protobuf/reflection.h" #include "google/protobuf/reflection_ops.h" @@ -179,6 +182,28 @@ struct is_internal_map_value_type : std::true_type {}; template <> struct is_internal_map_value_type : std::true_type {}; +struct MapTestPeer { + template + static typename T::KeyMapBase& GetKeyMapBase(T& value) { + return value; + } + + template + static bool InsertOrReplaceNode(T& map, typename T::key_type key, + typename T::mapped_type value) { + using Node = typename T::Node; + auto* node = static_cast(map.AllocNode(sizeof(Node))); + ::new (static_cast(&node->kv)) typename T::value_type{key, value}; + node = static_cast(GetKeyMapBase(map).InsertOrReplaceNode(node)); + if (node) { + node->~Node(); + GetKeyMapBase(map).DeallocNode(node, sizeof(Node)); + return false; + } + return true; + } +}; + namespace { using internal::DownCast; @@ -2738,6 +2763,42 @@ TEST(GeneratedMapFieldTest, Proto2UnknownEnum) { EXPECT_EQ(UNITTEST::PROTO2_MAP_ENUM_FOO, to.known_map_field().at(0)); } +TEST(GeneratedMapFieldTest, Proto2UnknownEnumAllKeyTypesWork) { +#define PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(Type, Value) \ + { \ + UNITTEST::TestEnumMapPlusExtra from; \ + auto value = Value; \ + ABSL_LOG(INFO) << "Testing " << #Type; \ + (*from.mutable_unknown_map_field_##Type())[Value] = \ + UNITTEST::E_PROTO2_MAP_ENUM_EXTRA; \ + UNITTEST::TestEnumMap to; \ + ASSERT_TRUE(to.ParseFromString(from.SerializeAsString())); \ + EXPECT_EQ(0, to.unknown_map_field_##Type().size()) \ + << testing::PrintToString(to.unknown_map_field_##Type()); \ + const UnknownFieldSet& unknown_field_set = \ + to.GetReflection()->GetUnknownFields(to); \ + EXPECT_EQ(1, unknown_field_set.field_count()); \ + from.Clear(); \ + EXPECT_EQ(from.unknown_map_field_##Type().size(), 0); \ + EXPECT_TRUE(from.ParseFromString(to.SerializeAsString())); \ + EXPECT_THAT(from.unknown_map_field_##Type(), \ + ElementsAre(Pair(value, UNITTEST::E_PROTO2_MAP_ENUM_EXTRA))); \ + } + + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(int64, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(uint64, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(int32, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(uint32, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(fixed32, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(fixed64, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(bool, true); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(string, "17"); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(sint32, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(sint64, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(sfixed32, 17); + PROTOBUF_INTERNAL_TEST_MAP_KEY_TYPE(sfixed64, 17); +} + TEST(GeneratedMapFieldTest, StandardWireFormat) { UNITTEST::TestMap message; std::string data = "\x0A\x04\x08\x01\x10\x01"; @@ -2910,6 +2971,41 @@ TEST(GeneratedMapFieldTest, UnknownFieldWireFormat) { EXPECT_EQ(3, message.map_int32_int32().at(2)); } +TEST(GeneratedMapFieldTest, ToplevelTagNotLengthPrefixed) { + UNITTEST::TestMap message; + + // The toplevel matches a field number, but does not match the wire type. + std::string data = "\x08\x81\x04"; + EXPECT_TRUE(message.ParseFromString(data)); + EXPECT_EQ(0, message.map_int32_int32().size()); + + const UnknownFieldSet& unknown_field_set = + message.GetReflection()->GetUnknownFields(message); + ASSERT_EQ(1, unknown_field_set.field_count()); + auto& field = unknown_field_set.field(0); + ASSERT_EQ(field.TYPE_VARINT, field.type()); + EXPECT_EQ(((0x1 << 0) | (0x04 << 7)), field.varint()); +} + +TEST(GeneratedMapFieldTest, InnerTagsInLongForm) { + UNITTEST::TestMap message; + + // First, control + absl::string_view data("\012\004\010\007\020\005", 6); + ASSERT_TRUE(message.ParseFromString(data)); + EXPECT_THAT(message.map_int32_int32(), ElementsAre(Pair(7, 5))); + + // Now we make the key tag long form + data = absl::string_view("\012\005\210\000\007\020\005", 7); + ASSERT_TRUE(message.ParseFromString(data)); + EXPECT_THAT(message.map_int32_int32(), ElementsAre(Pair(7, 5))); + + // Now we make the value tag long form + data = absl::string_view("\012\005\010\007\220\000\005", 7); + ASSERT_TRUE(message.ParseFromString(data)); + EXPECT_THAT(message.map_int32_int32(), ElementsAre(Pair(7, 5))); +} + TEST(GeneratedMapFieldTest, CorruptedWireFormat) { UNITTEST::TestMap message; @@ -3365,7 +3461,7 @@ TEST_F(MapFieldInDynamicMessageTest, MapValueReferernceValidAfterSerialize) { message->SerializeToString(&data); TestMap to; - to.ParseFromString(data); + ASSERT_TRUE(to.ParseFromString(data)); EXPECT_EQ(128, to.map_int32_foreign_message().at(0).c()); } @@ -4134,6 +4230,63 @@ TEST(MoveTest, MoveAssignmentWorks) { EXPECT_EQ(nested_msg43_ptr, &moved_to_map[43].optional_nested_message()); } +TEST(KeyMapBaseTest, InsertOrReplaceNodeWorks) { + using M = Map; + M map; + EXPECT_TRUE(MapTestPeer::InsertOrReplaceNode(map, 10, "Foo")); + EXPECT_EQ(map.size(), 1); + EXPECT_EQ(map[10], "Foo"); + EXPECT_FALSE(MapTestPeer::InsertOrReplaceNode(map, 10, "Bar")); + EXPECT_EQ(map.size(), 1); + EXPECT_EQ(map[10], "Bar"); + map[100] = "BAD"; + EXPECT_EQ(map.size(), 2); + EXPECT_EQ(map[10], "Bar"); + EXPECT_EQ(map[100], "BAD"); + EXPECT_FALSE(MapTestPeer::InsertOrReplaceNode(map, 100, "GOOD")); + EXPECT_EQ(map.size(), 2); + EXPECT_EQ(map[10], "Bar"); + EXPECT_EQ(map[100], "GOOD"); +} + +TEST(NonUtf8Test, StringValuePassesInProto2) { + protobuf_unittest::TestProto2BytesMap message; + (*message.mutable_map_string())[1] = "\xFF"; + + auto serialized = message.SerializeAsString(); + // Parsing passes, but a failure is logged in debug mode. + ASSERT_TRUE(message.ParseFromString(serialized)); + EXPECT_EQ((*message.mutable_map_string())[1], "\xFF"); +} + +TEST(NonUtf8Test, BytesValuePassesInProto2) { + protobuf_unittest::TestProto2BytesMap message; + (*message.mutable_map_bytes())[1] = "\xFF"; + + auto serialized = message.SerializeAsString(); + ASSERT_TRUE(message.ParseFromString(serialized)); + EXPECT_EQ((*message.mutable_map_bytes())[1], "\xFF"); +} + +TEST(NonUtf8Test, StringValueFailsInProto3) { + proto3_unittest::TestProto3BytesMap message; + (*message.mutable_map_string())[1] = "\xFF"; + + auto serialized = message.SerializeAsString(); + // It will fail, and log an error. + ASSERT_FALSE(message.ParseFromString(serialized)); +} + + +TEST(NonUtf8Test, BytesValuePassesInProto3) { + proto3_unittest::TestProto3BytesMap message; + (*message.mutable_map_bytes())[1] = "\xFF"; + + auto serialized = message.SerializeAsString(); + ASSERT_TRUE(message.ParseFromString(serialized)); + EXPECT_EQ((*message.mutable_map_bytes())[1], "\xFF"); +} + } // namespace } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index e3f8ae0c6718..a949eafaf808 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -516,6 +516,13 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { bool>::type = true> PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr); + // Read the length prefix, push the new limit, call the func(ptr), and then + // pop the limit. Useful for situations that don't value an actual message, + // like map entries. + template + PROTOBUF_NODISCARD const char* ParseLengthDelimitedInlined(const char*, + const Func& func); + template PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage( MessageLite* msg, const char* ptr, const Table* table) { @@ -648,6 +655,11 @@ T UnalignedLoad(const char* p) { memcpy(&res, &tmp, sizeof(T)); return res; } +template ::value>> +T UnalignedLoad(const Void* p) { + return UnalignedLoad(reinterpret_cast(p)); +} PROTOBUF_EXPORT std::pair VarintParseSlow32(const char* p, uint32_t res); @@ -944,6 +956,18 @@ PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg, return ptr; } +template +PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* +ParseContext::ParseLengthDelimitedInlined(const char* ptr, const Func& func) { + LimitToken old; + ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old); + if (ptr == nullptr) return ptr; + PROTOBUF_ALWAYS_INLINE_CALL ptr = func(ptr); + depth_++; + if (!PopLimit(std::move(old))) return nullptr; + return ptr; +} + inline const char* ParseContext::ReadSizeAndPushLimitAndDepthInlined( const char* ptr, LimitToken* old_limit) { int size = ReadSize(&ptr); diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 3283030c16dd..40fff1e7b207 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -261,6 +261,21 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and # define PROTOBUF_ALWAYS_INLINE #endif +#ifdef PROTOBUF_ALWAYS_INLINE_CALL +#error PROTOBUF_ALWAYS_INLINE_CALL was previously defined +#endif +// For functions we want to force inline from the caller, instead of in the +// declaration of the callee. +// This is useful for lambdas where it is not easy to specify ALWAYS_INLINE. +// Use like: +// PROTOBUF_ALWAYS_INLINE_CALL res = SomeFunc(args...); +#if defined(__clang__) && !defined(PROTOBUF_NO_INLINE_CALL) && \ + __has_cpp_attribute(clang::always_inline) +#define PROTOBUF_ALWAYS_INLINE_CALL [[clang::always_inline]] +#else +#define PROTOBUF_ALWAYS_INLINE_CALL +#endif + #ifdef PROTOBUF_NDEBUG_INLINE #error PROTOBUF_NDEBUG_INLINE was previously defined #endif diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index a0a5f648df40..1d2b1c16ba58 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -49,6 +49,7 @@ #undef PROTOBUF_NAMESPACE #undef PROTOBUF_NAMESPACE_ID #undef PROTOBUF_ALWAYS_INLINE +#undef PROTOBUF_ALWAYS_INLINE_CALL #undef PROTOBUF_NDEBUG_INLINE #undef PROTOBUF_MUSTTAIL #undef PROTOBUF_TAILCALL From 7fc47e2e60305d1ea560738fe23959a4bd07a557 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 Mar 2023 17:30:26 +0000 Subject: [PATCH 136/463] Auto-generate files after cl/513255347 --- src/file_lists.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index e331ac378398..0f933adb267a 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -618,6 +618,7 @@ set(protobuf_test_files set(protobuf_test_protos_files ${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.proto ${protobuf_SOURCE_DIR}/src/google/protobuf/map_proto2_unittest.proto + ${protobuf_SOURCE_DIR}/src/google/protobuf/map_proto3_unittest.proto ${protobuf_SOURCE_DIR}/src/google/protobuf/map_unittest.proto ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest.proto ${protobuf_SOURCE_DIR}/src/google/protobuf/unittest_arena.proto From dd8a3cf2b54a06ef0558c004f9fca570278ad4a1 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Mar 2023 10:04:51 -0800 Subject: [PATCH 137/463] Unconditionally generate unknown field accessors. This moves us closer to unifying proto2 and proto3 into editions. PiperOrigin-RevId: 513268016 --- src/google/protobuf/compiler/cpp/helpers.h | 5 ----- src/google/protobuf/compiler/cpp/message.cc | 18 ++++++++---------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index df9fba547460..c2af7c19f65d 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -323,11 +323,6 @@ std::string SafeFunctionName(const Descriptor* descriptor, const FieldDescriptor* field, absl::string_view prefix); -// Returns true if generated messages have public unknown fields accessors -inline bool PublicUnknownFieldsAccessors(const Descriptor* message) { - return message->file()->syntax() != FileDescriptor::SYNTAX_PROTO3; -} - // Returns the optimize mode for , respecting . FileOptions_OptimizeMode GetOptimizeFor(const FileDescriptor* file, const Options& options); diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index a9cb0545d0ff..82265929f452 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -1288,16 +1288,14 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* p) { "}\n" "\n"); - if (PublicUnknownFieldsAccessors(descriptor_)) { - format( - "inline const $unknown_fields_type$& unknown_fields() const {\n" - " return $unknown_fields$;\n" - "}\n" - "inline $unknown_fields_type$* mutable_unknown_fields() {\n" - " return $mutable_unknown_fields$;\n" - "}\n" - "\n"); - } + format( + "inline const $unknown_fields_type$& unknown_fields() const {\n" + " return $unknown_fields$;\n" + "}\n" + "inline $unknown_fields_type$* mutable_unknown_fields() {\n" + " return $mutable_unknown_fields$;\n" + "}\n" + "\n"); // Only generate this member if it's not disabled. if (HasDescriptorMethods(descriptor_->file(), options_) && From 5eabb720dd1654aedc53427730958876ea9932ad Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 Mar 2023 18:15:59 +0000 Subject: [PATCH 138/463] Auto-generate files after cl/513268016 --- src/google/protobuf/any.pb.h | 7 +++ src/google/protobuf/api.pb.h | 21 +++++++++ src/google/protobuf/duration.pb.h | 7 +++ src/google/protobuf/empty.pb.h | 7 +++ src/google/protobuf/field_mask.pb.h | 7 +++ src/google/protobuf/source_context.pb.h | 7 +++ src/google/protobuf/struct.pb.h | 21 +++++++++ src/google/protobuf/timestamp.pb.h | 7 +++ src/google/protobuf/type.pb.h | 35 ++++++++++++++ src/google/protobuf/wrappers.pb.h | 63 +++++++++++++++++++++++++ 10 files changed, 182 insertions(+) diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 493e9f439119..5d22bc1b61b0 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT Any final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 98332c726ece..8a91760d40b1 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -108,6 +108,13 @@ class PROTOBUF_EXPORT Api final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -379,6 +386,13 @@ class PROTOBUF_EXPORT Method final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -636,6 +650,13 @@ class PROTOBUF_EXPORT Mixin final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 35ea4f07b061..c68665caf77d 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT Duration final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index b4d58cf6577d..7c2687c1d7d7 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT Empty final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 7f835b317cb5..dba7515d2ea8 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT FieldMask final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index dced2d3fc6c7..69d5662b1e6c 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT SourceContext final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index f88800f3100d..ff41417d4604 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -171,6 +171,13 @@ class PROTOBUF_EXPORT Struct final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -332,6 +339,13 @@ class PROTOBUF_EXPORT Value final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -596,6 +610,13 @@ class PROTOBUF_EXPORT ListValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 3c0b4698807c..65b223b58b49 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -96,6 +96,13 @@ class PROTOBUF_EXPORT Timestamp final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index c89a53a356b5..98ca6fdcfe33 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -234,6 +234,13 @@ class PROTOBUF_EXPORT Type final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -493,6 +500,13 @@ class PROTOBUF_EXPORT Field final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -855,6 +869,13 @@ class PROTOBUF_EXPORT Enum final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -1084,6 +1105,13 @@ class PROTOBUF_EXPORT EnumValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -1273,6 +1301,13 @@ class PROTOBUF_EXPORT Option final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index ac85c6ff00c5..f5f845a20c30 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -136,6 +136,13 @@ class PROTOBUF_EXPORT DoubleValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -283,6 +290,13 @@ class PROTOBUF_EXPORT FloatValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -430,6 +444,13 @@ class PROTOBUF_EXPORT Int64Value final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -577,6 +598,13 @@ class PROTOBUF_EXPORT UInt64Value final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -724,6 +752,13 @@ class PROTOBUF_EXPORT Int32Value final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -871,6 +906,13 @@ class PROTOBUF_EXPORT UInt32Value final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -1018,6 +1060,13 @@ class PROTOBUF_EXPORT BoolValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -1165,6 +1214,13 @@ class PROTOBUF_EXPORT StringValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } @@ -1322,6 +1378,13 @@ class PROTOBUF_EXPORT BytesValue final : return *this; } + inline const ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet& unknown_fields() const { + return _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance); + } + inline ::PROTOBUF_NAMESPACE_ID::UnknownFieldSet* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } From 14d8ecfed2de3f0a07436d41c01036161a19d103 Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Wed, 1 Mar 2023 11:27:46 -0800 Subject: [PATCH 139/463] Internal change PiperOrigin-RevId: 513293094 --- src/google/protobuf/extension_set.h | 1 + src/google/protobuf/extension_set_unittest.cc | 1 - src/google/protobuf/generated_message_reflection_unittest.cc | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index a80bf9db2bc8..2178b1ea1ffd 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -62,6 +62,7 @@ #error "You cannot SWIG proto headers" #endif + namespace google { namespace protobuf { class Arena; diff --git a/src/google/protobuf/extension_set_unittest.cc b/src/google/protobuf/extension_set_unittest.cc index b94440429145..7f6799fa58e1 100644 --- a/src/google/protobuf/extension_set_unittest.cc +++ b/src/google/protobuf/extension_set_unittest.cc @@ -57,7 +57,6 @@ // Must be included last. #include "google/protobuf/port_def.inc" - namespace google { namespace protobuf { namespace internal { diff --git a/src/google/protobuf/generated_message_reflection_unittest.cc b/src/google/protobuf/generated_message_reflection_unittest.cc index f707cf5b95a3..3de242c2316c 100644 --- a/src/google/protobuf/generated_message_reflection_unittest.cc +++ b/src/google/protobuf/generated_message_reflection_unittest.cc @@ -64,7 +64,6 @@ // Must be included last. #include "google/protobuf/port_def.inc" - namespace google { namespace protobuf { From 5abab0f47e81ac085f0b2d17ec3b3a3b252a11f1 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 1 Mar 2023 13:01:46 -0800 Subject: [PATCH 140/463] Roll-forward of change to add Python support for retention attribute PiperOrigin-RevId: 513318052 --- .../protobuf/internal/generator_test.py | 77 +++++++++++++ .../protobuf/compiler/python/BUILD.bazel | 12 +- .../protobuf/compiler/python/generator.cc | 100 +++++++++-------- .../protobuf/compiler/python/generator.h | 11 +- src/google/protobuf/compiler/retention.cc | 103 ++++++++++++++---- src/google/protobuf/compiler/retention.h | 22 ++++ 6 files changed, 251 insertions(+), 74 deletions(-) diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 291b97d9328d..a4370acbfc2a 100644 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -49,6 +49,7 @@ from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_wire_format_pb2 from google.protobuf import unittest_pb2 +from google.protobuf import unittest_retention_pb2 from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_no_generic_services_pb2 @@ -152,6 +153,82 @@ def testMessageWithCustomOptions(self): # TODO(gps): We really should test for the presence of the enum_opt1 # extension and for its value to be set to -789. + # Options that are explicitly marked RETENTION_SOURCE should not be present + # in the descriptors in the binary. + def testOptionRetention(self): + # Direct options + options = unittest_retention_pb2.DESCRIPTOR.GetOptions() + self.assertTrue(options.HasExtension(unittest_retention_pb2.plain_option)) + self.assertTrue( + options.HasExtension(unittest_retention_pb2.runtime_retention_option) + ) + self.assertFalse( + options.HasExtension(unittest_retention_pb2.source_retention_option) + ) + + def check_options_message_is_stripped_correctly(options): + self.assertEqual(options.plain_field, 1) + self.assertEqual(options.runtime_retention_field, 2) + self.assertFalse(options.HasField('source_retention_field')) + self.assertEqual(options.source_retention_field, 0) + + # Verify that our test OptionsMessage is stripped correctly on all + # different entity types. + check_options_message_is_stripped_correctly( + options.Extensions[unittest_retention_pb2.file_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.GetOptions().Extensions[ + unittest_retention_pb2.message_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.NestedMessage.DESCRIPTOR.GetOptions().Extensions[ + unittest_retention_pb2.message_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELENUM.GetOptions().Extensions[ + unittest_retention_pb2.enum_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELMESSAGE_NESTEDENUM.GetOptions().Extensions[ + unittest_retention_pb2.enum_option + ] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2._TOPLEVELENUM.values[0] + .GetOptions() + .Extensions[unittest_retention_pb2.enum_entry_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.extensions_by_name['i'] + .GetOptions() + .Extensions[unittest_retention_pb2.field_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.fields[0] + .GetOptions() + .Extensions[unittest_retention_pb2.field_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.TopLevelMessage.DESCRIPTOR.oneofs[0] + .GetOptions() + .Extensions[unittest_retention_pb2.oneof_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] + .GetOptions() + .Extensions[unittest_retention_pb2.service_option] + ) + check_options_message_is_stripped_correctly( + unittest_retention_pb2.DESCRIPTOR.services_by_name['Service'] + .methods[0] + .GetOptions() + .Extensions[unittest_retention_pb2.method_option] + ) + def testNestedTypes(self): self.assertEqual( set(unittest_pb2.TestAllTypes.DESCRIPTOR.nested_types), diff --git a/src/google/protobuf/compiler/python/BUILD.bazel b/src/google/protobuf/compiler/python/BUILD.bazel index de1e06c8f355..90d5d21a8501 100644 --- a/src/google/protobuf/compiler/python/BUILD.bazel +++ b/src/google/protobuf/compiler/python/BUILD.bazel @@ -28,6 +28,7 @@ cc_library( deps = [ "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:retention", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -61,9 +62,12 @@ pkg_files( filegroup( name = "test_srcs", - srcs = glob([ - "*_test.cc", - "*unittest.cc", - ], allow_empty = True), + srcs = glob( + [ + "*_test.cc", + "*unittest.cc", + ], + allow_empty = True, + ), visibility = ["//src/google/protobuf/compiler:__pkg__"], ) diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index 7cc7b324d79d..651aaa991065 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -64,6 +64,7 @@ #include "absl/strings/substitute.h" #include "google/protobuf/compiler/python/helpers.h" #include "google/protobuf/compiler/python/pyi_generator.h" +#include "google/protobuf/compiler/retention.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -249,8 +250,7 @@ bool Generator::Generate(const FileDescriptor* file, std::string filename = GetFileName(file, ".py"); - FileDescriptorProto fdp; - file_->CopyTo(&fdp); + FileDescriptorProto fdp = StripSourceRetentionOptions(*file_); fdp.SerializeToString(&file_descriptor_serialized_); if (!opensource_runtime_ && GeneratingDescriptorProto()) { @@ -342,7 +342,7 @@ bool Generator::Generate(const FileDescriptor* file, FixAllDescriptorOptions(); // Set serialized_start and serialized_end. - SetSerializedPbInterval(); + SetSerializedPbInterval(fdp); printer_->Outdent(); if (HasGenericServices(file)) { @@ -441,7 +441,8 @@ void Generator::PrintFileDescriptor() const { m["name"] = file_->name(); m["package"] = file_->package(); m["syntax"] = StringifySyntax(file_->syntax()); - m["options"] = OptionsValue(file_->options().SerializeAsString()); + m["options"] = OptionsValue( + StripLocalSourceRetentionOptions(*file_).SerializeAsString()); m["serialized_descriptor"] = absl::CHexEscape(file_descriptor_serialized_); if (GeneratingDescriptorProto()) { printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); @@ -527,7 +528,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { " create_key=_descriptor._internal_create_key,\n" " values=[\n"; std::string options_string; - enum_descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(enum_descriptor) + .SerializeToString(&options_string); printer_->Print(m, enum_descriptor_template); printer_->Indent(); printer_->Indent(); @@ -680,7 +682,8 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { printer_->Outdent(); printer_->Print("],\n"); std::string options_string; - message_descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(message_descriptor) + .SerializeToString(&options_string); printer_->Print( "serialized_options=$options_value$,\n" "is_extendable=$extendable$,\n" @@ -707,7 +710,8 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { m["name"] = desc->name(); m["full_name"] = desc->full_name(); m["index"] = absl::StrCat(desc->index()); - options_string = OptionsValue(desc->options().SerializeAsString()); + options_string = OptionsValue( + StripLocalSourceRetentionOptions(*desc).SerializeAsString()); if (options_string == "None") { m["serialized_options"] = ""; } else { @@ -1049,7 +1053,8 @@ void Generator::PrintEnumValueDescriptor( // TODO(robinson): Fix up EnumValueDescriptor "type" fields. // More circular references. ::sigh:: std::string options_string; - descriptor.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(descriptor) + .SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = descriptor.name(); m["index"] = absl::StrCat(descriptor.index()); @@ -1077,7 +1082,7 @@ std::string Generator::OptionsValue( void Generator::PrintFieldDescriptor(const FieldDescriptor& field, bool is_extension) const { std::string options_string; - field.options().SerializeToString(&options_string); + StripLocalSourceRetentionOptions(field).SerializeToString(&options_string); absl::flat_hash_map m; m["name"] = field.name(); m["full_name"] = field.full_name(); @@ -1205,21 +1210,17 @@ std::string Generator::ModuleLevelServiceDescriptorName( return name; } -// Prints standard constructor arguments serialized_start and serialized_end. +// Prints descriptor offsets _serialized_start and _serialized_end. // Args: -// descriptor: The cpp descriptor to have a serialized reference. -// proto: A proto +// descriptor_proto: The descriptor proto to have a serialized reference. // Example printer output: -// serialized_start=41, -// serialized_end=43, -// -template -void Generator::PrintSerializedPbInterval(const DescriptorT& descriptor, - DescriptorProtoT& proto, - absl::string_view name) const { - descriptor.CopyTo(&proto); +// _globals['_MYMESSAGE']._serialized_start=47 +// _globals['_MYMESSAGE']._serialized_end=76 +template +void Generator::PrintSerializedPbInterval( + const DescriptorProtoT& descriptor_proto, absl::string_view name) const { std::string sp; - proto.SerializeToString(&sp); + descriptor_proto.SerializeToString(&sp); int offset = file_descriptor_serialized_.find(sp); ABSL_CHECK_GE(offset, 0); @@ -1243,43 +1244,47 @@ void PrintDescriptorOptionsFixingCode(absl::string_view descriptor, } } // namespace -void Generator::SetSerializedPbInterval() const { +// Generates the start and end offsets for each entity in the serialized file +// descriptor. The file argument must exactly match what was serialized into +// file_descriptor_serialized_, and should already have had any +// source-retention options stripped out. This is important because we need an +// exact byte-for-byte match so that we can successfully find the correct +// offsets in the serialized descriptors. +void Generator::SetSerializedPbInterval(const FileDescriptorProto& file) const { // Top level enums. for (int i = 0; i < file_->enum_type_count(); ++i) { - EnumDescriptorProto proto; const EnumDescriptor& descriptor = *file_->enum_type(i); - PrintSerializedPbInterval(descriptor, proto, + PrintSerializedPbInterval(file.enum_type(i), ModuleLevelDescriptorName(descriptor)); } // Messages. for (int i = 0; i < file_->message_type_count(); ++i) { - SetMessagePbInterval(*file_->message_type(i)); + SetMessagePbInterval(file.message_type(i), *file_->message_type(i)); } // Services. for (int i = 0; i < file_->service_count(); ++i) { - ServiceDescriptorProto proto; const ServiceDescriptor& service = *file_->service(i); - PrintSerializedPbInterval(service, proto, + PrintSerializedPbInterval(file.service(i), ModuleLevelServiceDescriptorName(service)); } } -void Generator::SetMessagePbInterval(const Descriptor& descriptor) const { - DescriptorProto message_proto; - PrintSerializedPbInterval(descriptor, message_proto, +void Generator::SetMessagePbInterval(const DescriptorProto& message_proto, + const Descriptor& descriptor) const { + PrintSerializedPbInterval(message_proto, ModuleLevelDescriptorName(descriptor)); // Nested messages. for (int i = 0; i < descriptor.nested_type_count(); ++i) { - SetMessagePbInterval(*descriptor.nested_type(i)); + SetMessagePbInterval(message_proto.nested_type(i), + *descriptor.nested_type(i)); } for (int i = 0; i < descriptor.enum_type_count(); ++i) { - EnumDescriptorProto proto; const EnumDescriptor& enum_des = *descriptor.enum_type(i); - PrintSerializedPbInterval(enum_des, proto, + PrintSerializedPbInterval(message_proto.enum_type(i), ModuleLevelDescriptorName(enum_des)); } } @@ -1287,7 +1292,8 @@ void Generator::SetMessagePbInterval(const Descriptor& descriptor) const { // Prints expressions that set the options field of all descriptors. void Generator::FixAllDescriptorOptions() const { // Prints an expression that sets the file descriptor's options. - std::string file_options = OptionsValue(file_->options().SerializeAsString()); + std::string file_options = OptionsValue( + StripLocalSourceRetentionOptions(*file_).SerializeAsString()); if (file_options != "None") { PrintDescriptorOptionsFixingCode(kDescriptorKey, file_options, printer_); } else { @@ -1315,7 +1321,8 @@ void Generator::FixAllDescriptorOptions() const { } void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { - std::string oneof_options = OptionsValue(oneof.options().SerializeAsString()); + std::string oneof_options = + OptionsValue(StripLocalSourceRetentionOptions(oneof).SerializeAsString()); if (oneof_options != "None") { std::string oneof_name = absl::Substitute( "$0.$1['$2']", ModuleLevelDescriptorName(*oneof.containing_type()), @@ -1328,15 +1335,15 @@ void Generator::FixOptionsForOneof(const OneofDescriptor& oneof) const { // value descriptors. void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const { std::string descriptor_name = ModuleLevelDescriptorName(enum_descriptor); - std::string enum_options = - OptionsValue(enum_descriptor.options().SerializeAsString()); + std::string enum_options = OptionsValue( + StripLocalSourceRetentionOptions(enum_descriptor).SerializeAsString()); if (enum_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, enum_options, printer_); } for (int i = 0; i < enum_descriptor.value_count(); ++i) { const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(i); - std::string value_options = - OptionsValue(value_descriptor.options().SerializeAsString()); + std::string value_options = OptionsValue( + StripLocalSourceRetentionOptions(value_descriptor).SerializeAsString()); if (value_options != "None") { PrintDescriptorOptionsFixingCode( absl::StrFormat("%s.values_by_name[\"%s\"]", descriptor_name.c_str(), @@ -1352,8 +1359,8 @@ void Generator::FixOptionsForService( const ServiceDescriptor& service_descriptor) const { std::string descriptor_name = ModuleLevelServiceDescriptorName(service_descriptor); - std::string service_options = - OptionsValue(service_descriptor.options().SerializeAsString()); + std::string service_options = OptionsValue( + StripLocalSourceRetentionOptions(service_descriptor).SerializeAsString()); if (service_options != "None") { PrintDescriptorOptionsFixingCode(descriptor_name, service_options, printer_); @@ -1361,8 +1368,8 @@ void Generator::FixOptionsForService( for (int i = 0; i < service_descriptor.method_count(); ++i) { const MethodDescriptor* method = service_descriptor.method(i); - std::string method_options = - OptionsValue(method->options().SerializeAsString()); + std::string method_options = OptionsValue( + StripLocalSourceRetentionOptions(*method).SerializeAsString()); if (method_options != "None") { std::string method_name = absl::StrCat( descriptor_name, ".methods_by_name['", method->name(), "']"); @@ -1374,7 +1381,8 @@ void Generator::FixOptionsForService( // Prints expressions that set the options for field descriptors (including // extensions). void Generator::FixOptionsForField(const FieldDescriptor& field) const { - std::string field_options = OptionsValue(field.options().SerializeAsString()); + std::string field_options = + OptionsValue(StripLocalSourceRetentionOptions(field).SerializeAsString()); if (field_options != "None") { std::string field_name; if (field.is_extension()) { @@ -1419,8 +1427,8 @@ void Generator::FixOptionsForMessage(const Descriptor& descriptor) const { FixOptionsForField(field); } // Message option for this message. - std::string message_options = - OptionsValue(descriptor.options().SerializeAsString()); + std::string message_options = OptionsValue( + StripLocalSourceRetentionOptions(descriptor).SerializeAsString()); if (message_options != "None") { std::string descriptor_name = ModuleLevelDescriptorName(descriptor); PrintDescriptorOptionsFixingCode(descriptor_name, message_options, diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index c203cfcba8a4..65c16f591eb0 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -41,6 +41,7 @@ #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/descriptor.pb.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -164,9 +165,8 @@ class PROTOC_EXPORT Generator : public CodeGenerator { std::string ModuleLevelServiceDescriptorName( const ServiceDescriptor& descriptor) const; - template - void PrintSerializedPbInterval(const DescriptorT& descriptor, - DescriptorProtoT& proto, + template + void PrintSerializedPbInterval(const DescriptorProtoT& descriptor_proto, absl::string_view name) const; void FixAllDescriptorOptions() const; @@ -176,8 +176,9 @@ class PROTOC_EXPORT Generator : public CodeGenerator { void FixOptionsForService(const ServiceDescriptor& descriptor) const; void FixOptionsForMessage(const Descriptor& descriptor) const; - void SetSerializedPbInterval() const; - void SetMessagePbInterval(const Descriptor& descriptor) const; + void SetSerializedPbInterval(const FileDescriptorProto& file) const; + void SetMessagePbInterval(const DescriptorProto& message_proto, + const Descriptor& descriptor) const; void CopyPublicDependenciesAliases(absl::string_view copy_from, const FileDescriptor* file) const; diff --git a/src/google/protobuf/compiler/retention.cc b/src/google/protobuf/compiler/retention.cc index 087af7260c1a..04937a40e644 100644 --- a/src/google/protobuf/compiler/retention.cc +++ b/src/google/protobuf/compiler/retention.cc @@ -42,6 +42,7 @@ namespace protobuf { namespace compiler { namespace { + // Recursively strips any options with source retention from the message. void StripMessage(Message& m) { const Reflection* reflection = m.GetReflection(); @@ -62,43 +63,107 @@ void StripMessage(Message& m) { } } } -} // namespace -FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { - FileDescriptorProto file_proto; - file.CopyTo(&file_proto); - - // We need to look up the descriptor in file.pool() so that we can get a +// Converts the descriptor to a dynamic message if necessary, and then strips +// out all source-retention options. +// +// The options message may have custom options set on it, and these would +// ordinarily appear as unknown fields since they are not linked into protoc. +// Using a dynamic message allows us to see these custom options. To convert +// back and forth between the generated type and the dynamic message, we have +// to serialize one and parse that into the other. +void ConvertToDynamicMessageAndStripOptions(Message& m, + const DescriptorPool& pool) { + // We need to look up the descriptor in the pool so that we can get a // descriptor which knows about any custom options that were used in the // .proto file. - const Descriptor* descriptor = - file.pool()->FindMessageTypeByName(FileDescriptorProto().GetTypeName()); + const Descriptor* descriptor = pool.FindMessageTypeByName(m.GetTypeName()); if (descriptor == nullptr) { - // If the pool does not contain the descriptor for FileDescriptorProto, - // then this proto file does not transitively depend on descriptor.proto, - // in which case we know there are no custom options to worry about. - StripMessage(file_proto); + // If the pool does not contain the descriptor, then this proto file does + // not transitively depend on descriptor.proto, in which case we know there + // are no custom options to worry about. + StripMessage(m); } else { - // The options message may have custom options set on it, and these would - // ordinarily appear as unknown fields since they are not linked into - // protoc. Using a dynamic message allows us to see these custom options. - // To convert back and forth between the generated type and the dynamic - // message, we have to serialize one and parse that into the other. DynamicMessageFactory factory; std::unique_ptr dynamic_message( factory.GetPrototype(descriptor)->New()); std::string serialized; - ABSL_CHECK(file_proto.SerializeToString(&serialized)); + ABSL_CHECK(m.SerializeToString(&serialized)); ABSL_CHECK(dynamic_message->ParseFromString(serialized)); StripMessage(*dynamic_message); ABSL_CHECK(dynamic_message->SerializeToString(&serialized)); - ABSL_CHECK(file_proto.ParseFromString(serialized)); + ABSL_CHECK(m.ParseFromString(serialized)); } +} + +// Returns a const reference to the descriptor pool associated with the given +// descriptor. +template +const google::protobuf::DescriptorPool& GetPool(const DescriptorType& descriptor) { + return *descriptor.file()->pool(); +} + +// Specialization for FileDescriptor. +const google::protobuf::DescriptorPool& GetPool(const FileDescriptor& descriptor) { + return *descriptor.pool(); +} + +// Returns the options associated with the given descriptor, with all +// source-retention options stripped out. +template +auto StripLocalOptions(const DescriptorType& descriptor) { + auto options = descriptor.options(); + ConvertToDynamicMessageAndStripOptions(options, GetPool(descriptor)); + return options; +} +} // namespace + +FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file) { + FileDescriptorProto file_proto; + file.CopyTo(&file_proto); + ConvertToDynamicMessageAndStripOptions(file_proto, *file.pool()); return file_proto; } +EnumOptions StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +EnumValueOptions StripLocalSourceRetentionOptions( + const EnumValueDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +FieldOptions StripLocalSourceRetentionOptions( + const FieldDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +FileOptions StripLocalSourceRetentionOptions(const FileDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +MessageOptions StripLocalSourceRetentionOptions(const Descriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +MethodOptions StripLocalSourceRetentionOptions( + const MethodDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +OneofOptions StripLocalSourceRetentionOptions( + const OneofDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + +ServiceOptions StripLocalSourceRetentionOptions( + const ServiceDescriptor& descriptor) { + return StripLocalOptions(descriptor); +} + } // namespace compiler } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/compiler/retention.h b/src/google/protobuf/compiler/retention.h index b97ab1e73a9e..0579b067e7f8 100644 --- a/src/google/protobuf/compiler/retention.h +++ b/src/google/protobuf/compiler/retention.h @@ -46,6 +46,28 @@ namespace compiler { PROTOC_EXPORT FileDescriptorProto StripSourceRetentionOptions(const FileDescriptor& file); +// The following functions take a descriptor and strip all source-retention +// options from just the local entity (e.g. message, enum, field). Most code +// generators should not need these functions, but they are sometimes useful if +// you need to strip the options on a single entity rather than handling the +// entire file at once. +PROTOC_EXPORT EnumOptions +StripLocalSourceRetentionOptions(const EnumDescriptor& descriptor); +PROTOC_EXPORT EnumValueOptions +StripLocalSourceRetentionOptions(const EnumValueDescriptor& descriptor); +PROTOC_EXPORT FieldOptions +StripLocalSourceRetentionOptions(const FieldDescriptor& descriptor); +PROTOC_EXPORT FileOptions +StripLocalSourceRetentionOptions(const FileDescriptor& descriptor); +PROTOC_EXPORT MessageOptions +StripLocalSourceRetentionOptions(const Descriptor& descriptor); +PROTOC_EXPORT MethodOptions +StripLocalSourceRetentionOptions(const MethodDescriptor& descriptor); +PROTOC_EXPORT OneofOptions +StripLocalSourceRetentionOptions(const OneofDescriptor& descriptor); +PROTOC_EXPORT ServiceOptions +StripLocalSourceRetentionOptions(const ServiceDescriptor& descriptor); + } // namespace compiler } // namespace protobuf } // namespace google From 5888882691665bf701ccc4e9d463d1a41d15ec3e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Mar 2023 13:45:10 -0800 Subject: [PATCH 141/463] Internal change PiperOrigin-RevId: 513329841 --- BUILD.bazel | 2 +- build_defs/BUILD.bazel | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index bc229ddf671f..b0b29f812883 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -179,7 +179,7 @@ cc_binary( linkopts = LINK_OPTS, features = select({ # This isn't possible on mac because there is no static library for lcrt0.o - "@platforms//os:osx": [], + "//build_defs:config_osx": [], # When cross-compiling we need to statically link all C++ libraries. "//conditions:default": ["fully_static_link"], }), diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel index 4ac09c82989f..0926b856d83e 100644 --- a/build_defs/BUILD.bazel +++ b/build_defs/BUILD.bazel @@ -1,5 +1,6 @@ # Internal Starlark definitions for Protobuf. +load("@bazel_skylib//lib:selects.bzl", "selects") load("@rules_cc//cc:defs.bzl", starlark_cc_proto_library = "cc_proto_library") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") @@ -83,6 +84,25 @@ config_setting( }, ) +config_setting( + name = "config_osx_aarch64", + values = {"cpu": "osx-aarch_64"}, +) + +config_setting( + name = "config_osx_x86_64", + values = {"cpu": "osx-aarch_64"}, +) + +selects.config_setting_group( + name = "config_osx", + match_any = [ + "@platforms//os:osx", + ":config_osx_aarch64", + ":config_osx_x86_64", + ] +) + # Internal testing: starlark_cc_proto_library( From af791f28a8217d84c6ba14ee2063f7b3f5230171 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Mar 2023 14:59:04 -0800 Subject: [PATCH 142/463] Add libm linkage to toolchains PiperOrigin-RevId: 513350744 --- toolchain/cc_toolchain_config.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl index b610a8ee9d96..3bfe1bfa9337 100644 --- a/toolchain/cc_toolchain_config.bzl +++ b/toolchain/cc_toolchain_config.bzl @@ -98,6 +98,7 @@ def _impl(ctx): flags = [ "-B" + ctx.attr.linker_path, "-lstdc++", + "-lm", "--target=" + ctx.attr.target_full_name, ] + ctx.attr.extra_linker_flags, ), From a66f75dfeb25c9f9917a318f5f6a541ecb7e0336 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Mar 2023 15:31:28 -0800 Subject: [PATCH 143/463] Migrate syntax reflection to new feature-based system. PiperOrigin-RevId: 513358980 --- src/google/protobuf/compiler/cpp/helpers.cc | 4 ---- src/google/protobuf/compiler/cpp/helpers.h | 6 ------ src/google/protobuf/compiler/cpp/message.cc | 17 +++-------------- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index 7b1414955e60..cd4919cdf315 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -1092,10 +1092,6 @@ bool ShouldVerify(const FileDescriptor* file, const Options& options, return false; } -bool IsUtf8String(const FieldDescriptor* field) { - return IsProto3(field->file()) && - field->type() == FieldDescriptor::TYPE_STRING; -} VerifySimpleType ShouldVerifySimple(const Descriptor* descriptor) { (void)descriptor; diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index c2af7c19f65d..6a97d1f18053 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -465,10 +465,6 @@ bool IsStringOrMessage(const FieldDescriptor* field); std::string UnderscoresToCamelCase(absl::string_view input, bool cap_next_letter); -inline bool IsProto3(const FileDescriptor* file) { - return file->syntax() == FileDescriptor::SYNTAX_PROTO3; -} - inline bool IsCrossFileMessage(const FieldDescriptor* field) { return field->type() == FieldDescriptor::TYPE_MESSAGE && field->message_type()->file() != field->file(); @@ -1045,8 +1041,6 @@ enum class VerifySimpleType { // Returns VerifySimpleType if messages can be verified by predefined methods. VerifySimpleType ShouldVerifySimple(const Descriptor* descriptor); -bool IsUtf8String(const FieldDescriptor* field); - bool HasMessageFieldOrExtension(const Descriptor* desc); // Generates a vector of substitutions for use with Printer::WithVars that diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 82265929f452..e467f1e5ae4d 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -252,18 +252,6 @@ bool EmitFieldNonDefaultCondition(io::Printer* p, const std::string& prefix, return false; } -// Does the given field have a has_$name$() method? -bool HasHasMethod(const FieldDescriptor* field) { - if (!IsProto3(field->file())) { - // In proto1/proto2, every field has a has_$name$() method. - return true; - } - // For message types without true field presence, only fields with a message - // type or inside an one-of have a has_$name$() method. - return field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE || - field->has_optional_keyword() || field->real_containing_oneof(); -} - bool HasInternalHasMethod(const FieldDescriptor* field) { return !HasHasbit(field) && field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE; @@ -1040,7 +1028,7 @@ void MessageGenerator::GenerateOneofMemberHasBits(const FieldDescriptor* field, // N.B.: Without field presence, we do not use has-bits or generate // has_$name$() methods, but oneofs still have set_has_$name$(). // Oneofs also have private _internal_has_$name$() a helper method. - if (HasHasMethod(field)) { + if (field->has_presence()) { format( "inline bool $classname$::has_$name$() const {\n" "$annotate_has$" @@ -3651,7 +3639,8 @@ void MessageGenerator::GenerateSerializeWithCachedSizesBody(io::Printer* p) { LazySerializerEmitter(MessageGenerator* mg, io::Printer* p) : mg_(mg), p_(p), - eager_(IsProto3(mg->descriptor_->file())), + eager_(mg->descriptor_->file()->syntax() == + FileDescriptor::SYNTAX_PROTO3), cached_has_bit_index_(kNoHasbit) {} ~LazySerializerEmitter() { Flush(); } From 34b7d583bdaa4ef9f8fd76906fb80cc8a0eb3847 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 2 Mar 2023 10:56:34 -0800 Subject: [PATCH 144/463] Internal change PiperOrigin-RevId: 513584095 --- src/google/protobuf/compiler/java/file.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/compiler/java/file.cc b/src/google/protobuf/compiler/java/file.cc index e90bbfe6a008..adaf2a69ff90 100644 --- a/src/google/protobuf/compiler/java/file.cc +++ b/src/google/protobuf/compiler/java/file.cc @@ -51,6 +51,7 @@ #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/compiler/java/service.h" #include "google/protobuf/compiler/java/shared_code_generator.h" +#include "google/protobuf/compiler/retention.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/io/printer.h" @@ -452,8 +453,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable( // To find those extensions, we need to parse the data into a dynamic message // of the FileDescriptor based on the builder-pool, then we can use // reflections to find all extension fields - FileDescriptorProto file_proto; - file_->CopyTo(&file_proto); + FileDescriptorProto file_proto = StripSourceRetentionOptions(*file_); std::string file_data; file_proto.SerializeToString(&file_data); FieldDescriptorSet extensions; @@ -521,8 +521,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable( // Check if custom options exist. If any, try to load immutable classes since // custom options are only represented with immutable messages. - FileDescriptorProto file_proto; - file_->CopyTo(&file_proto); + FileDescriptorProto file_proto = StripSourceRetentionOptions(*file_); std::string file_data; file_proto.SerializeToString(&file_data); FieldDescriptorSet extensions; From 0a9282b82b675ffad2566aceed1db8fdf96db734 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 Mar 2023 13:39:14 -0800 Subject: [PATCH 145/463] Reduce stack frame size of `TextFormat::Printer::PrintUnknownFields` by offloading `std::string` temporaries into a separate function. PiperOrigin-RevId: 513627351 --- src/google/protobuf/text_format.cc | 27 ++++++++++++--------- src/google/protobuf/text_format.h | 7 +++++- src/google/protobuf/text_format_unittest.cc | 17 +++++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index ac199007faf7..9d213a361c58 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -2699,18 +2699,23 @@ void TextFormat::Printer::PrintFieldValue(const Message& message, return Parser().ParseFieldValueFromString(input, field, message); } +template +PROTOBUF_NOINLINE void TextFormat::OutOfLinePrintString( + BaseTextGenerator* generator, const T&... values) { + generator->PrintString(absl::StrCat(values...)); +} + void TextFormat::Printer::PrintUnknownFields( const UnknownFieldSet& unknown_fields, BaseTextGenerator* generator, int recursion_budget) const { for (int i = 0; i < unknown_fields.field_count(); i++) { const UnknownField& field = unknown_fields.field(i); - std::string field_number = absl::StrCat(field.number()); switch (field.type()) { case UnknownField::TYPE_VARINT: - generator->PrintString(field_number); + OutOfLinePrintString(generator, field.number()); generator->PrintMaybeWithMarker(MarkerToken(), ": "); - generator->PrintString(absl::StrCat(field.varint())); + OutOfLinePrintString(generator, field.varint()); if (single_line_mode_) { generator->PrintLiteral(" "); } else { @@ -2718,10 +2723,10 @@ void TextFormat::Printer::PrintUnknownFields( } break; case UnknownField::TYPE_FIXED32: { - generator->PrintString(field_number); + OutOfLinePrintString(generator, field.number()); generator->PrintMaybeWithMarker(MarkerToken(), ": ", "0x"); - generator->PrintString( - absl::StrCat(absl::Hex(field.fixed32(), absl::kZeroPad8))); + OutOfLinePrintString(generator, + absl::Hex(field.fixed32(), absl::kZeroPad8)); if (single_line_mode_) { generator->PrintLiteral(" "); } else { @@ -2730,10 +2735,10 @@ void TextFormat::Printer::PrintUnknownFields( break; } case UnknownField::TYPE_FIXED64: { - generator->PrintString(field_number); + OutOfLinePrintString(generator, field.number()); generator->PrintMaybeWithMarker(MarkerToken(), ": ", "0x"); - generator->PrintString( - absl::StrCat(absl::Hex(field.fixed64(), absl::kZeroPad16))); + OutOfLinePrintString(generator, + absl::Hex(field.fixed64(), absl::kZeroPad16)); if (single_line_mode_) { generator->PrintLiteral(" "); } else { @@ -2742,7 +2747,7 @@ void TextFormat::Printer::PrintUnknownFields( break; } case UnknownField::TYPE_LENGTH_DELIMITED: { - generator->PrintString(field_number); + OutOfLinePrintString(generator, field.number()); const std::string& value = field.length_delimited(); // We create a CodedInputStream so that we can adhere to our recursion // budget when we attempt to parse the data. UnknownFieldSet parsing is @@ -2783,7 +2788,7 @@ void TextFormat::Printer::PrintUnknownFields( break; } case UnknownField::TYPE_GROUP: - generator->PrintString(field_number); + OutOfLinePrintString(generator, field.number()); if (single_line_mode_) { generator->PrintMaybeWithMarker(MarkerToken(), " ", "{ "); } else { diff --git a/src/google/protobuf/text_format.h b/src/google/protobuf/text_format.h index 122c8a4472f7..74f954b7cb21 100644 --- a/src/google/protobuf/text_format.h +++ b/src/google/protobuf/text_format.h @@ -141,7 +141,7 @@ class PROTOBUF_EXPORT TextFormat { // Print text to the output stream. virtual void Print(const char* text, size_t size) = 0; - void PrintString(const std::string& str) { Print(str.data(), str.size()); } + void PrintString(absl::string_view str) { Print(str.data(), str.size()); } template void PrintLiteral(const char (&text)[n]) { @@ -724,6 +724,11 @@ class PROTOBUF_EXPORT TextFormat { ParseLocationRange location); static inline ParseInfoTree* CreateNested(ParseInfoTree* info_tree, const FieldDescriptor* field); + // To reduce stack frame bloat we use an out-of-line function to print + // strings. This avoid local std::string temporaries. + template + static void OutOfLinePrintString(BaseTextGenerator* generator, + const T&... values); }; diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index f17185531f2a..7920accad6bc 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -297,6 +297,22 @@ TEST_F(TextFormatTest, PrintUnknownFields) { message.DebugString()); } +TEST_F(TextFormatTest, PrintUnknownFieldsDeepestStackWorks) { + // Test printing of unknown fields in a message. + + unittest::TestEmptyMessage message; + UnknownFieldSet* unknown_fields = message.mutable_unknown_fields(); + + for (int i = 0; i < 200; ++i) { + unknown_fields = unknown_fields->AddGroup(1); + } + + unknown_fields->AddVarint(2, 100); + + std::string str; + EXPECT_TRUE(TextFormat::PrintToString(message, &str)); +} + TEST_F(TextFormatTest, PrintUnknownFieldsHidden) { // Test printing of unknown fields in a message when suppressed. @@ -2350,6 +2366,7 @@ TEST(TextFormatFloatingPointTest, PreservesNegative0) { std::signbit(out_message.optional_double())); } + } // namespace text_format_unittest } // namespace protobuf } // namespace google From ac8725ce21141f4bcfccf9ee49389c2cc8dae36c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 Mar 2023 15:12:22 -0800 Subject: [PATCH 146/463] Fix false negative in the verifier where it was accepting a non-canonical zero field number. PiperOrigin-RevId: 513652171 --- src/google/protobuf/message_unittest.inc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index f5e330e0a498..914e3e9f334e 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -333,6 +333,13 @@ TEST(MESSAGE_TEST_NAME, ParseFailNonCanonicalZeroTag) { absl::string_view{encoded, sizeof(encoded) - 1})); } +TEST(MESSAGE_TEST_NAME, ParseFailNonCanonicalZeroField) { + const char encoded[] = {"\012\x6\205\0\0\0\0\0"}; + UNITTEST::NestedTestAllTypes parsed; + EXPECT_FALSE(parsed.ParsePartialFromString( + absl::string_view{encoded, sizeof(encoded) - 1})); +} + TEST(MESSAGE_TEST_NAME, NestedExplicitLazyExceedRecursionLimit) { UNITTEST::NestedTestAllTypes original, parsed; // Build proto with recursion depth of 5, with nested annotated LazyField. From dcb21556cc20417a7bc6d1577f3c4065d28ec69b Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Mar 2023 15:54:26 -0800 Subject: [PATCH 147/463] Migrate PHP Extension tests to shared CI infrastructure. This will reduce most of the flakes we've seen from these tests. PiperOrigin-RevId: 513662586 --- .github/workflows/test_php_ext.yml | 65 ++++++++++++++++-------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test_php_ext.yml b/.github/workflows/test_php_ext.yml index 901512c13bd4..993e0da9b4e6 100644 --- a/.github/workflows/test_php_ext.yml +++ b/.github/workflows/test_php_ext.yml @@ -12,45 +12,50 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - build-php: - name: Build + package: + name: Package runs-on: ubuntu-latest - container: ${{ matrix.php-image }} - strategy: - matrix: - php-image: - - php:7.4-cli - - php:8.1-cli - # TODO(b/266868629) Dockerize these instead of installing all the - # dependencies on each run. steps: - - name: Install python3 - run: | - apt-get update -q - apt-get install -qy python3 - - name: Install bazel - run: | - apt-get install -qy wget - mkdir $HOME/bin - wget -O $HOME/bin/bazel https://github.com/bazelbuild/bazel/releases/download/5.3.2/bazel-5.3.2-linux-x86_64 - chmod a+x $HOME/bin/bazel - - name: Install git - run: | - apt-get install -qy --no-install-recommends git - name: Checkout uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - - name: Create package - run: | - cd $GITHUB_WORKSPACE - rm -rf bazel-bin/php/protobuf-*.tgz - $HOME/bin/bazel build php:release + + - name: Package extension + uses: protocolbuffers/protobuf-ci/bazel@v1 + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: php_ext/${{ matrix.version }} + bash: > + bazel build //php:release $BAZEL_FLAGS; + cp bazel-bin/php/protobuf-*.tgz . + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: protobuf-php-release + path: protobuf-*.tgz + + build: + needs: [package] + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + include: + - php-image: php:7.4-cli + version: "7.4.18-dbg" + - php-image: php:8.1-cli + version: "8.1.14" + name: Build ${{ matrix.version }} + runs-on: ubuntu-latest + container: ${{ matrix.php-image }} + steps: + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + with: + name: protobuf-php-release - name: Compile extension run: | cd /tmp - MAKE="make -j$(nproc)" pecl install $GITHUB_WORKSPACE/bazel-bin/php/protobuf-*.tgz + MAKE="make -j$(nproc)" pecl install $GITHUB_WORKSPACE/protobuf-*.tgz - name: Enable extension run: docker-php-ext-enable protobuf - name: Inspect extension From a54ae5ea9a272cd9acd838de072209f0e2e08a3e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 3 Mar 2023 07:25:15 -0800 Subject: [PATCH 148/463] Internal Change PiperOrigin-RevId: 513821856 --- src/google/protobuf/generated_message_reflection.cc | 1 + src/google/protobuf/message.h | 1 + src/google/protobuf/port_def.inc | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index ed5a868ff71c..7df03b316e6a 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -1760,6 +1760,7 @@ const std::string& Reflection::GetStringReference(const Message& message, } + void Reflection::SetString(Message* message, const FieldDescriptor* field, std::string value) const { USAGE_CHECK_ALL(SetString, SINGULAR, STRING); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index ffd639ed04df..fac5c2438334 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -630,6 +630,7 @@ class PROTOBUF_EXPORT Reflection final { std::string* scratch) const; + // Singular field mutators ----------------------------------------- // These mutate the value of a non-repeated field. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 40fff1e7b207..10fb6645aacf 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -213,6 +213,10 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // Owner: shaod@, gberg@ #define PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL 1 +// Enable cord handling. +// Owner: mvels@, mkruskal@ +#define PROTOBUF_FUTURE_OPENSOURCE_CORD 1 + #endif #ifdef PROTOBUF_VERSION From eaa8f24aa626bf0432f8d137eed8e070850a6d98 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Fri, 3 Mar 2023 08:57:13 -0800 Subject: [PATCH 149/463] Updated obsolete Python README.md PiperOrigin-RevId: 513840426 --- python/README.md | 235 +++++++++++++++++++++-------------------------- 1 file changed, 105 insertions(+), 130 deletions(-) diff --git a/python/README.md b/python/README.md index 6549fb67000b..dc4b1d07ffb3 100644 --- a/python/README.md +++ b/python/README.md @@ -1,130 +1,105 @@ -Protocol Buffers - Google's data interchange format -=================================================== - -Copyright 2008 Google Inc. - -This directory contains the Python Protocol Buffers runtime library. - -Normally, this directory comes as part of the protobuf package, available -from: - - https://developers.google.com/protocol-buffers/ - -The complete package includes the C++ source code, which includes the -Protocol Compiler (protoc). If you downloaded this package from PyPI -or some other Python-specific source, you may have received only the -Python part of the code. In this case, you will need to obtain the -Protocol Compiler from some other source before you can use this -package. - -Development Warning -=================== - -The pure python performance is slow. For better performance please -use python c++ implementation. - -Installation -============ - -1) Make sure you have Python 3.7 or newer. If in doubt, run: - - $ python -V - -2) Make sure you have Bazel 0.5.4 or later (or CMake 3.5 or later). - -3) If you do not have setuptools installed, note that it will be - downloaded and installed automatically as soon as you run `setup.py`. - If you would rather install it manually, you may do so by following - the instructions on [this page](https://packaging.python.org/en/latest/tutorials/installing-packages/). - -4) Build the C++ code, or install a binary distribution of `protoc`. If - you install a binary distribution, make sure that it is the same - version as this package. If in doubt, run: - - $ protoc --version - -5) Build and run the tests: - - $ python setup.py build - $ python setup.py test - - To build, test, and use the C++ implementation, you must first compile - `libprotobuf.so` using either [Bazel](../README.md) or [CMake](../src/README.md): - - On OS X: - - If you are running a Homebrew-provided Python, you must make sure another - version of protobuf is not already installed, as Homebrew's Python will - search `/usr/local/lib` for `libprotobuf.so` before it searches the compiled - binaries. - - You can either unlink Homebrew's protobuf or install the `libprotobuf` you - built earlier: - - $ brew unlink protobuf - - or - - $ (cd .. && cmake . && make install) - - On other *nix: - - You must make `libprotobuf.so` dynamically available. You can either - install libprotobuf you built earlier, or set `LD_LIBRARY_PATH`: - - $ (cd .. && cmake . && make -j20 install) - - or - - $ export LD_LIBRARY_PATH=../bazel-bin - - To build the C++ implementation run: - - $ python setup.py build --cpp_implementation - - Then run the tests like so: - - $ python setup.py test --cpp_implementation - - If some tests fail, this library may not work correctly on your - system. Continue at your own risk. - - Please note that there is a known problem with some versions of - Python on Cygwin which causes the tests to fail after printing the - error: `sem_init: Resource temporarily unavailable`. This appears - to be a [bug either in Cygwin or in - Python](http://www.cygwin.com/ml/cygwin/2005-07/msg01378.html). - - We do not know if or when it might be fixed. We also do not know - how likely it is that this bug will affect users in practice. - -6) Install: - - $ python setup.py install - - or: - - $ (cd .. && make install) - $ python setup.py install --cpp_implementation - - This step may require superuser privileges. - NOTE: To use C++ implementation, you need to export an environment - variable before running your program. See the "C++ Implementation" - section below for more details. - -Usage -===== - -The complete documentation for Protocol Buffers is available via the -web at: - - https://developers.google.com/protocol-buffers/ - -C++ Implementation -================== - -The C++ implementation for Python messages is built as a Python extension to -improve the overall protobuf Python performance. - -To use the C++ implementation, you need to install the C++ protobuf runtime -library, please see instructions in the parent directory. +# Protocol Buffers Python + +This directory contains the Protobuf library for Python. + +In most cases you should install the library using `pip` or another package +manager: + +``` +$ pip install protobuf +``` + +The packages released on https://pypi.org/project/protobuf/#files include both a +source distribution and binary wheels. + +For user documentation about how to use Protobuf Python, see +https://protobuf.dev/getting-started/pythontutorial/ + +# Building packages from this repo + +If for some reason you wish to build the packages directly from this repo, you +can use the following Bazel commands: + +``` +$ bazel build @upb//python/dist:source_wheel +$ bazel build @upb//python/dist:binary_wheel +``` + +The binary wheel will build against whatever version of Python is installed on +your system. The source package is always the same and does not depend on a +local version of Python. + +# Implementation backends + +There are three separate implementations of Python Protobuf. All of them offer +the same API and are thus functionally the same, though they have very different +performance characteristics. + +The runtime library contains a switching layer that can choose between these +backends at runtime. Normally it will choose between them automatically, using +priority-ordered list, and skipping any backends that are not available. However +you can request a specific backend by setting the +`PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION` environment variable to one of the +following values: + +1. **upb**: Built on the + [upb C library](https://github.com/protocolbuffers/upb), this is a new + extension module + [released in 4.21.0](https://protobuf.dev/news/2022-05-06/). It offers + better performance than any of the previous backends, and it is now the + default. It is distributed in our PyPI packages, and requires no special + installation. The code for this module lives in + [@upb/python](https://github.com/protocolbuffers/upb/tree/main/python). +1. **cpp**: This extension module wraps the C++ protobuf library. It is + deprecated and is no longer released in our PyPI packages, however it is + still used in some legacy cases where apps want to perform zero-copy message + sharing between Python and C++. It must be installed separately before it + can be used. The code for this module lives in + [google/protobuf/pyext](https://github.com/protocolbuffers/protobuf/tree/main/python/google/protobuf/pyext). +1. **python**: The pure-Python backend, this does not require any extension + module to be present on the system. The code for the pure-Python backend + lives in [google/protobuf/internal](google/protobuf/internal) + +The order given above is the general priority order, with `upb` being preferred +the most and the `python` backend preferred the least. However this ordering can +be overridden by the presence of a +`google.protobuf.internal._api_implementation` module. See the logic in +[api_implementation.py](https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/api_implementation.py) +for details. + +You can check which backend you are using with the following snippet: + +``` +$ python +Python 3.10.9 (main, Dec 7 2022, 13:47:07) [GCC 12.2.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> from google.protobuf.internal import api_implementation +>>> print(api_implementation.Type()) +upb +``` + +This is not an officially-supported or stable API, but it is useful for ad hoc +diagnostics. + +More information about sharing messages between Python and C++ is available +here: https://protobuf.dev/reference/python/python-generated/#sharing-messages + +# Code generator + +The code for the Protobuf Python code generator lives in +[//src/google/protobuf/compiler/python](https://github.com/protocolbuffers/protobuf/tree/main/src/google/protobuf/compiler/python). +The code generator can output two different files for each proto `foo.proto`: + +* **foo_pb2.py**: The module you import to actually use the protos. +* **foo_pb2.pyi**: A stub file that describes the interface of the protos. + +The `foo_pb2.pyi` file is useful for IDEs or for users who want to read the +output file. The `foo_pb2.py` file is optimized for fast loading and is not +readable at all. + +Note that the pyi file is only generated if you pass the `pyi_out` option to +`protoc`: + +``` +$ protoc --python_out=pyi_out:output_dir +``` From fab7f928b5375a20fd8d33556632128e936ad436 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 3 Mar 2023 09:52:58 -0800 Subject: [PATCH 150/463] Expand LazySerializerEmitter to cover proto3 cases. This should provide some performance wins for proto3, similar to the ones added years ago for proto2. PiperOrigin-RevId: 513853367 --- src/google/protobuf/compiler/cpp/message.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index e467f1e5ae4d..e34f19bde369 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -3639,8 +3639,6 @@ void MessageGenerator::GenerateSerializeWithCachedSizesBody(io::Printer* p) { LazySerializerEmitter(MessageGenerator* mg, io::Printer* p) : mg_(mg), p_(p), - eager_(mg->descriptor_->file()->syntax() == - FileDescriptor::SYNTAX_PROTO3), cached_has_bit_index_(kNoHasbit) {} ~LazySerializerEmitter() { Flush(); } @@ -3649,13 +3647,14 @@ void MessageGenerator::GenerateSerializeWithCachedSizesBody(io::Printer* p) { // oneof, and handle them at the next Flush(). void Emit(const FieldDescriptor* field) { Formatter format(p_); - if (eager_ || MustFlush(field)) { + + if (!field->has_presence() || MustFlush(field)) { Flush(); } if (!field->real_containing_oneof()) { // TODO(ckennelly): Defer non-oneof fields similarly to oneof fields. - if (!field->options().weak() && !field->is_repeated() && !eager_) { + if (HasHasbit(field) && field->has_presence()) { // We speculatively load the entire _has_bits_[index] contents, even // if it is for only one field. Deferring non-oneof emitting would // allow us to determine whether this is going to be useful. @@ -3699,7 +3698,6 @@ void MessageGenerator::GenerateSerializeWithCachedSizesBody(io::Printer* p) { MessageGenerator* mg_; io::Printer* p_; - bool eager_; std::vector v_; // cached_has_bit_index_ maintains that: From e83848f09288693d0fe9a71824908bb8f621bd63 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 18:07:21 +0000 Subject: [PATCH 151/463] Auto-generate files after cl/513853367 --- src/google/protobuf/struct.pb.cc | 81 +++++++++++++++----------------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 47e930ee1ebc..b1f1f4af1f7b 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -701,49 +701,46 @@ ::uint8_t* Value::_InternalSerialize( ::uint32_t cached_has_bits = 0; (void) cached_has_bits; - // .google.protobuf.NullValue null_value = 1; - if (kind_case() == kNullValue) { - target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteEnumToArray( - 1, this->_internal_null_value(), target); - } - - // double number_value = 2; - if (kind_case() == kNumberValue) { - target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteDoubleToArray( - 2, this->_internal_number_value(), target); - } - - // string string_value = 3; - if (kind_case() == kStringValue) { - const std::string& _s = this->_internal_string_value(); - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - _s.data(), static_cast(_s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "google.protobuf.Value.string_value"); - target = stream->WriteStringMaybeAliased(3, _s, target); - } - - // bool bool_value = 4; - if (kind_case() == kBoolValue) { - target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteBoolToArray( - 4, this->_internal_bool_value(), target); - } - - // .google.protobuf.Struct struct_value = 5; - if (kind_case() == kStructValue) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessage(5, _Internal::struct_value(this), - _Internal::struct_value(this).GetCachedSize(), target, stream); - } - - // .google.protobuf.ListValue list_value = 6; - if (kind_case() == kListValue) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessage(6, _Internal::list_value(this), - _Internal::list_value(this).GetCachedSize(), target, stream); + switch (kind_case()) { + case kNullValue: { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteEnumToArray( + 1, this->_internal_null_value(), target); + break; + } + case kNumberValue: { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteDoubleToArray( + 2, this->_internal_number_value(), target); + break; + } + case kStringValue: { + const std::string& _s = this->_internal_string_value(); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + _s.data(), static_cast(_s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "google.protobuf.Value.string_value"); + target = stream->WriteStringMaybeAliased(3, _s, target); + break; + } + case kBoolValue: { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteBoolToArray( + 4, this->_internal_bool_value(), target); + break; + } + case kStructValue: { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(5, _Internal::struct_value(this), + _Internal::struct_value(this).GetCachedSize(), target, stream); + break; + } + case kListValue: { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(6, _Internal::list_value(this), + _Internal::list_value(this).GetCachedSize(), target, stream); + break; + } + default: ; } - if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); From 797b470fe8f0b77301aeac7e4d09c81edd853844 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 12:13:32 -0800 Subject: [PATCH 152/463] Automated rollback of commit bf0d8e238480bdbe5d0136bd3a6186b2569f8f2f. PiperOrigin-RevId: 513889737 --- src/google/protobuf/descriptor.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index a321f94f863e..efcd86e35895 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3740,10 +3740,10 @@ class DescriptorBuilder { // Maximum recursion depth corresponds to 32 nested message declarations. int recursion_depth_ = 32; - void AddError(absl::string_view element_name, const Message& descriptor, + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - absl::string_view error); - void AddError(absl::string_view element_name, const Message& descriptor, + const std::string& error); + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error); void AddRecursiveImportError(const FileDescriptorProto& proto, int from_here); @@ -3937,7 +3937,6 @@ class DescriptorBuilder { void SuggestFieldNumbers(FileDescriptor* file, const FileDescriptorProto& proto); - // Must be run only after cross-linking. void InterpretOptions(); @@ -4190,9 +4189,9 @@ DescriptorBuilder::DescriptorBuilder( DescriptorBuilder::~DescriptorBuilder() {} PROTOBUF_NOINLINE void DescriptorBuilder::AddError( - absl::string_view element_name, const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - absl::string_view error) { + const std::string& error) { if (error_collector_ == nullptr) { if (!had_errors_) { ABSL_LOG(ERROR) << "Invalid proto descriptor for file \"" << filename_ @@ -4207,7 +4206,7 @@ PROTOBUF_NOINLINE void DescriptorBuilder::AddError( } PROTOBUF_NOINLINE void DescriptorBuilder::AddError( - absl::string_view element_name, const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error) { AddError(element_name, descriptor, location, std::string(error)); } @@ -6463,7 +6462,6 @@ void DescriptorBuilder::CrossLinkExtensionRange( } - void DescriptorBuilder::CrossLinkField(FieldDescriptor* field, const FieldDescriptorProto& proto) { if (field->options_ == nullptr) { From ae2531dcc2492c51554ea36d15540ff816ca6abd Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 3 Mar 2023 12:20:37 -0800 Subject: [PATCH 153/463] Make protoc --descriptor_set_out respect option retention PiperOrigin-RevId: 513891493 --- cmake/tests.cmake | 6 ++- src/google/protobuf/compiler/BUILD.bazel | 13 +++++-- .../compiler/command_line_interface.cc | 3 +- .../command_line_interface_unittest.cc | 37 +++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 27f2eaca7297..26080604663d 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -131,7 +131,8 @@ target_link_libraries(lite-test ) add_test(NAME lite-test - COMMAND lite-test ${protobuf_GTEST_ARGS}) + COMMAND lite-test ${protobuf_GTEST_ARGS} + WORKING_DIRECTORY ${protobuf_SOURCE_DIR}) add_custom_target(full-test COMMAND tests @@ -139,7 +140,8 @@ add_custom_target(full-test WORKING_DIRECTORY ${protobuf_SOURCE_DIR}) add_test(NAME full-test - COMMAND tests ${protobuf_GTEST_ARGS}) + COMMAND tests ${protobuf_GTEST_ARGS} + WORKING_DIRECTORY ${protobuf_SOURCE_DIR}) # For test purposes, remove headers that should already be installed. This # prevents accidental conflicts and also version skew (since local headers take diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index b8d9e2d3db40..af50b87c604e 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -88,6 +88,7 @@ cc_library( deps = [ ":code_generator", ":importer", + ":retention", "//src/google/protobuf:protobuf_nowkt", "@com_google_absl//absl/container:btree", "@com_google_absl//absl/log:absl_check", @@ -255,6 +256,7 @@ cc_test( data = [ ":test_plugin", "//:test_proto_srcs", + "//src/google/protobuf:descriptor_proto_srcs", "//src/google/protobuf:testdata", ], deps = [ @@ -374,10 +376,13 @@ pkg_files( filegroup( name = "test_srcs", - srcs = glob([ - "*_test.cc", - "*unittest.cc", - ], allow_empty = True) + [ + srcs = glob( + [ + "*_test.cc", + "*unittest.cc", + ], + allow_empty = True, + ) + [ "//src/google/protobuf/compiler/cpp:test_srcs", "//src/google/protobuf/compiler/csharp:test_srcs", "//src/google/protobuf/compiler/java:test_srcs", diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 43a576d952af..effdd3a66f1e 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -86,6 +86,7 @@ #include "absl/strings/substitute.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/importer.h" +#include "google/protobuf/compiler/retention.h" #include "google/protobuf/compiler/zip_writer.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/dynamic_message.h" @@ -2631,7 +2632,7 @@ void CommandLineInterface::GetTransitiveDependencies( // Add this file. FileDescriptorProto* new_descriptor = output->Add(); - file->CopyTo(new_descriptor); + *new_descriptor = StripSourceRetentionOptions(*file); if (include_json_name) { file->CopyJsonNameTo(new_descriptor); } diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index c89f47345375..0b6d4aafed77 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -1693,6 +1693,43 @@ TEST_F(CommandLineInterfaceTest, WriteTransitiveDescriptorSetWithSourceInfo) { EXPECT_TRUE(descriptor_set.file(1).has_source_code_info()); } +TEST_F(CommandLineInterfaceTest, DescriptorSetOptionRetention) { + // clang-format off + CreateTempFile( + "foo.proto", + absl::Substitute(R"pb( + syntax = "proto2"; + import "$0"; + extend google.protobuf.FileOptions { + optional int32 runtime_retention_option = 50001 + [retention = RETENTION_RUNTIME]; + optional int32 source_retention_option = 50002 + [retention = RETENTION_SOURCE]; + } + option (runtime_retention_option) = 2; + option (source_retention_option) = 3;)pb", + DescriptorProto::descriptor()->file()->name())); + // clang-format on + std::string descriptor_proto_base_dir = "src"; + Run(absl::Substitute( + "protocol_compiler --descriptor_set_out=$$tmpdir/descriptor_set " + "--proto_path=$$tmpdir --proto_path=$0 foo.proto", + descriptor_proto_base_dir)); + ExpectNoErrors(); + + FileDescriptorSet descriptor_set; + ReadDescriptorSet("descriptor_set", &descriptor_set); + ASSERT_EQ(descriptor_set.file_size(), 1); + const UnknownFieldSet& unknown_fields = + descriptor_set.file(0).options().unknown_fields(); + + // We expect runtime_retention_option to be present while + // source_retention_option should have been stripped. + ASSERT_EQ(unknown_fields.field_count(), 1); + EXPECT_EQ(unknown_fields.field(0).number(), 50001); + EXPECT_EQ(unknown_fields.field(0).varint(), 2); +} + #ifdef _WIN32 // TODO(teboring): Figure out how to write test on windows. #else From ad77e1ab2f6b14550ff990542dcfbc013394d47a Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 3 Mar 2023 13:13:52 -0800 Subject: [PATCH 154/463] Migrate syntax reflection to new feature-based system. PiperOrigin-RevId: 513904931 --- src/google/protobuf/compiler/java/enum.cc | 8 +-- .../protobuf/compiler/java/enum_field.cc | 50 +++++++++---------- .../protobuf/compiler/java/enum_field_lite.cc | 46 ++++++++--------- .../protobuf/compiler/java/enum_lite.cc | 6 +-- src/google/protobuf/compiler/java/helpers.cc | 16 +++++- src/google/protobuf/compiler/java/helpers.h | 26 ++++------ .../protobuf/compiler/java/map_field.cc | 42 ++++++---------- .../protobuf/compiler/java/map_field_lite.cc | 46 +++++++---------- .../protobuf/compiler/java/message_lite.cc | 2 +- .../protobuf/compiler/java/primitive_field.cc | 16 +++--- .../compiler/java/primitive_field_lite.cc | 18 +++---- .../protobuf/compiler/java/string_field.cc | 16 +++--- .../compiler/java/string_field_lite.cc | 18 +++---- 13 files changed, 148 insertions(+), 162 deletions(-) diff --git a/src/google/protobuf/compiler/java/enum.cc b/src/google/protobuf/compiler/java/enum.cc index 84c99e6779a4..f64d89e91331 100644 --- a/src/google/protobuf/compiler/java/enum.cc +++ b/src/google/protobuf/compiler/java/enum.cc @@ -119,7 +119,7 @@ void EnumGenerator::Generate(io::Printer* printer) { printer->Annotate("name", canonical_values_[i]); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { if (ordinal_is_index) { printer->Print("${$UNRECOGNIZED$}$(-1),\n", "{", "", "}", ""); } else { @@ -167,7 +167,7 @@ void EnumGenerator::Generate(io::Printer* printer) { printer->Print( "\n" "public final int getNumber() {\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { if (ordinal_is_index) { printer->Print( " if (this == UNRECOGNIZED) {\n" @@ -251,7 +251,7 @@ void EnumGenerator::Generate(io::Printer* printer) { printer->Print( "public final com.google.protobuf.Descriptors.EnumValueDescriptor\n" " getValueDescriptor() {\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { if (ordinal_is_index) { printer->Print( " if (this == UNRECOGNIZED) {\n" @@ -346,7 +346,7 @@ void EnumGenerator::Generate(io::Printer* printer) { " \"EnumValueDescriptor is not for this type.\");\n" " }\n", "classname", descriptor_->name()); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { printer->Print( " if (desc.getIndex() == -1) {\n" " return UNRECOGNIZED;\n" diff --git a/src/google/protobuf/compiler/java/enum_field.cc b/src/google/protobuf/compiler/java/enum_field.cc index 59d56d4a35ab..c45f1a4c2a3a 100644 --- a/src/google/protobuf/compiler/java/enum_field.cc +++ b/src/google/protobuf/compiler/java/enum_field.cc @@ -88,7 +88,7 @@ void SetEnumVariables( ? absl::StrCat("@kotlin.Deprecated(message = \"Field ", (*variables)["name"], " is deprecated\") ") : ""}); - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); // Note that these have a trailing ";". @@ -120,7 +120,7 @@ void SetEnumVariables( (*variables)["get_has_field_bit_from_local"] = GenerateGetBitFromLocal(builderBitIndex); - if (SupportUnknownEnumValue(descriptor->file())) { + if (SupportUnknownEnumValue(descriptor)) { variables->insert( {"unknown", absl::StrCat((*variables)["type"], ".UNRECOGNIZED")}); } else { @@ -162,12 +162,12 @@ int ImmutableEnumFieldGenerator::GetNumBitsForBuilder() const { return 1; } void ImmutableEnumFieldGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean has$capitalized_name$();\n"); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print(variables_, "$deprecation$int get$capitalized_name$Value();\n"); @@ -179,7 +179,7 @@ void ImmutableEnumFieldGenerator::GenerateInterfaceMembers( void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const { printer->Print(variables_, "private int $name$_ = $default_number$;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override $deprecation$public boolean " @@ -188,7 +188,7 @@ void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const { "}\n"); printer->Annotate("{", "}", descriptor_); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print(variables_, "@java.lang.Override $deprecation$public int " @@ -210,7 +210,7 @@ void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const { void ImmutableEnumFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { printer->Print(variables_, "private int $name$_ = $default_number$;\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override $deprecation$public boolean " @@ -219,7 +219,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print(variables_, "@java.lang.Override $deprecation$public int " @@ -286,7 +286,7 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" @@ -305,7 +305,7 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -333,12 +333,12 @@ void ImmutableEnumFieldGenerator::GenerateBuilderClearCode( void ImmutableEnumFieldGenerator::GenerateMergingCode( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { printer->Print(variables_, "if (other.has$capitalized_name$()) {\n" " set$capitalized_name$(other.get$capitalized_name$());\n" "}\n"); - } else if (SupportUnknownEnumValue(descriptor_->file())) { + } else if (SupportUnknownEnumValue(descriptor_)) { printer->Print( variables_, "if (other.$name$_ != $default_number$) {\n" @@ -362,7 +362,7 @@ void ImmutableEnumFieldGenerator::GenerateBuildingCode( void ImmutableEnumFieldGenerator::GenerateBuilderParsingCode( io::Printer* printer) const { - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print(variables_, "$name$_ = input.readEnum();\n" "$set_has_field_bit_builder$\n"); @@ -429,7 +429,7 @@ ImmutableEnumOneofFieldGenerator::~ImmutableEnumOneofFieldGenerator() {} void ImmutableEnumOneofFieldGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$public boolean ${$has$capitalized_name$$}$() {\n" @@ -437,7 +437,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -464,7 +464,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMembers( void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -473,7 +473,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -552,7 +552,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuildingCode( void ImmutableEnumOneofFieldGenerator::GenerateMergingCode( io::Printer* printer) const { - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print( variables_, "set$capitalized_name$Value(other.get$capitalized_name$Value());\n"); @@ -564,7 +564,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMergingCode( void ImmutableEnumOneofFieldGenerator::GenerateBuilderParsingCode( io::Printer* printer) const { - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print(variables_, "int rawValue = input.readEnum();\n" "$set_oneof_case_message$;\n" @@ -604,7 +604,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateSerializedSizeCode( void ImmutableEnumOneofFieldGenerator::GenerateEqualsCode( io::Printer* printer) const { - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print( variables_, "if (get$capitalized_name$Value()\n" @@ -619,7 +619,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateEqualsCode( void ImmutableEnumOneofFieldGenerator::GenerateHashCode( io::Printer* printer) const { - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print(variables_, "hash = (37 * hash) + $constant_name$;\n" "hash = (53 * hash) + get$capitalized_name$Value();\n"); @@ -661,7 +661,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateInterfaceMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_GETTER); printer->Print(variables_, "$deprecation$$type$ get$capitalized_name$(int index);\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "$deprecation$java.util.List\n" @@ -716,7 +716,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( " return $name$_converter_.convert($name$_.get(index));\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "@java.lang.Override\n" @@ -844,7 +844,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "$deprecation$public java.util.List\n" @@ -954,7 +954,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuildingCode( void RepeatedImmutableEnumFieldGenerator::GenerateBuilderParsingCode( io::Printer* printer) const { // Read and store the enum - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print(variables_, "int tmpRaw = input.readEnum();\n" "ensure$capitalized_name$IsMutable();\n" diff --git a/src/google/protobuf/compiler/java/enum_field_lite.cc b/src/google/protobuf/compiler/java/enum_field_lite.cc index fc3cb1fa3f2a..011f5c7a06f2 100644 --- a/src/google/protobuf/compiler/java/enum_field_lite.cc +++ b/src/google/protobuf/compiler/java/enum_field_lite.cc @@ -97,7 +97,7 @@ void SetEnumVariables( : ""}); (*variables)["required"] = descriptor->is_required() ? "true" : "false"; - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { if (!context->options().opensource_runtime) { (*variables)["bit_field_id"] = absl::StrCat(messageBitIndex / 32); (*variables)["bit_field_name"] = GetBitFieldNameForBit(messageBitIndex); @@ -128,7 +128,7 @@ void SetEnumVariables( (*variables)["set_has_field_bit_to_local"] = GenerateSetBitToLocal(messageBitIndex); - if (SupportUnknownEnumValue(descriptor->file())) { + if (SupportUnknownEnumValue(descriptor)) { variables->insert( {"unknown", absl::StrCat((*variables)["type"], ".UNRECOGNIZED")}); } else { @@ -166,13 +166,13 @@ int ImmutableEnumFieldLiteGenerator::GetNumBitsForMessage() const { void ImmutableEnumFieldLiteGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean ${$has$capitalized_name$$}$();\n"); printer->Annotate("{", "}", descriptor_); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print(variables_, "$deprecation$int ${$get$capitalized_name$Value$}$();\n"); @@ -193,7 +193,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateMembers( " fieldNumber=$number$,\n" " type=com.google.protobuf.FieldType.$annotation_field_type$,\n" " isRequired=$required$)\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" " presenceBitsId=$bit_field_id$,\n" @@ -202,7 +202,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateMembers( } printer->Print(variables_, "private int $name$_;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -212,7 +212,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateMembers( "}\n"); printer->Annotate("{", "}", descriptor_); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -232,7 +232,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateMembers( printer->Annotate("{", "}", descriptor_); // Generate private setters for the builder to proxy into. - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, SETTER); printer->Print(variables_, "private void set$capitalized_name$Value(int value) {\n" @@ -256,7 +256,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateMembers( void ImmutableEnumFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -266,7 +266,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -327,7 +327,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { printer->Print( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" @@ -346,7 +346,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -369,7 +369,7 @@ void ImmutableEnumFieldLiteGenerator::GenerateFieldInfo( WriteIntToUtf16CharSequence(descriptor_->number(), output); WriteIntToUtf16CharSequence(GetExperimentalJavaFieldType(descriptor_), output); - if (HasHasbit(descriptor_)) { + if (descriptor_->has_presence()) { WriteIntToUtf16CharSequence(messageBitIndex_, output); } printer->Print(variables_, "\"$name$_\",\n"); @@ -400,7 +400,7 @@ ImmutableEnumOneofFieldLiteGenerator::~ImmutableEnumOneofFieldLiteGenerator() {} void ImmutableEnumOneofFieldLiteGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -409,7 +409,7 @@ void ImmutableEnumOneofFieldLiteGenerator::GenerateMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -436,7 +436,7 @@ void ImmutableEnumOneofFieldLiteGenerator::GenerateMembers( printer->Annotate("{", "}", descriptor_); // Generate private setters for the builder to proxy into. - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, SETTER); printer->Print(variables_, "private void set$capitalized_name$Value(int value) {\n" @@ -476,7 +476,7 @@ void ImmutableEnumOneofFieldLiteGenerator::GenerateFieldInfo( void ImmutableEnumOneofFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -485,7 +485,7 @@ void ImmutableEnumOneofFieldLiteGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, GETTER); printer->Print( variables_, @@ -571,7 +571,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateInterfaceMembers( variables_, "$deprecation$$type$ ${$get$capitalized_name$$}$(int index);\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "$deprecation$java.util.List\n" @@ -641,7 +641,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateMembers( " return result == null ? $unknown$ : result;\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "@java.lang.Override\n" @@ -707,7 +707,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateMembers( " $name$_ = emptyIntList();\n" "}\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, SETTER); printer->Print(variables_, "private void set$capitalized_name$Value(\n" @@ -740,7 +740,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateFieldInfo( WriteIntToUtf16CharSequence(GetExperimentalJavaFieldType(descriptor_), output); printer->Print(variables_, "\"$name$_\",\n"); - if (!SupportUnknownEnumValue(descriptor_->file())) { + if (!SupportUnknownEnumValue(descriptor_)) { PrintEnumVerifierLogic(printer, descriptor_, variables_, /*var_name=*/"$type$", /*terminating_string=*/",\n", @@ -815,7 +815,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(descriptor_)) { WriteFieldEnumValueAccessorDocComment(printer, descriptor_, LIST_GETTER); printer->Print(variables_, "@java.lang.Override\n" diff --git a/src/google/protobuf/compiler/java/enum_lite.cc b/src/google/protobuf/compiler/java/enum_lite.cc index 5d10a684bc37..abadc78df9c1 100644 --- a/src/google/protobuf/compiler/java/enum_lite.cc +++ b/src/google/protobuf/compiler/java/enum_lite.cc @@ -97,7 +97,7 @@ void EnumLiteGenerator::Generate(io::Printer* printer) { printer->Annotate("name", canonical_values_[i]); } - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { printer->Print("${$UNRECOGNIZED$}$(-1),\n", "{", "", "}", ""); printer->Annotate("{", "}", descriptor_); } @@ -142,7 +142,7 @@ void EnumLiteGenerator::Generate(io::Printer* printer) { "\n" "@java.lang.Override\n" "public final int getNumber() {\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { printer->Print( " if (this == UNRECOGNIZED) {\n" " throw new java.lang.IllegalArgumentException(\n" @@ -235,7 +235,7 @@ void EnumLiteGenerator::Generate(io::Printer* printer) { " result.append(getClass().getName()).append('@')\n" " .append(java.lang.Integer.toHexString(\n" " java.lang.System.identityHashCode(this)));\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (!descriptor_->is_closed()) { printer->Print( " if (this != UNRECOGNIZED) {\n" " result.append(\" number=\").append(getNumber());\n" diff --git a/src/google/protobuf/compiler/java/helpers.cc b/src/google/protobuf/compiler/java/helpers.cc index 8c3784d2e686..eef40ee3ba7b 100644 --- a/src/google/protobuf/compiler/java/helpers.cc +++ b/src/google/protobuf/compiler/java/helpers.cc @@ -932,7 +932,7 @@ int GetExperimentalJavaFieldType(const FieldDescriptor* field) { } if (field->is_map()) { - if (!SupportUnknownEnumValue(field)) { + if (!SupportUnknownEnumValue(MapValueField(field))) { const FieldDescriptor* value = field->message_type()->map_value(); if (GetJavaType(value) == JAVATYPE_ENUM) { extra_bits |= kMapWithProto2EnumValue; @@ -977,6 +977,20 @@ void EscapeUtf16ToString(uint16_t code, std::string* output) { } } +const FieldDescriptor* MapKeyField(const FieldDescriptor* descriptor) { + ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); + const Descriptor* message = descriptor->message_type(); + ABSL_CHECK(message->options().map_entry()); + return message->map_key(); +} + +const FieldDescriptor* MapValueField(const FieldDescriptor* descriptor) { + ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); + const Descriptor* message = descriptor->message_type(); + ABSL_CHECK(message->options().map_entry()); + return message->map_value(); +} + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index 0ebb3b9ebe6a..185c89c21895 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -354,28 +354,19 @@ inline bool HasPackedFields(const Descriptor* descriptor) { // them has a required field. Return true if a required field is found. bool HasRequiredFields(const Descriptor* descriptor); -inline bool IsProto2(const FileDescriptor* descriptor) { - return descriptor->syntax() == FileDescriptor::SYNTAX_PROTO2; -} - inline bool IsRealOneof(const FieldDescriptor* descriptor) { return descriptor->containing_oneof() && !descriptor->containing_oneof()->is_synthetic(); } -inline bool HasHazzer(const FieldDescriptor* descriptor) { - return !descriptor->is_repeated() && - (descriptor->message_type() || descriptor->has_optional_keyword() || - IsProto2(descriptor->file()) || IsRealOneof(descriptor)); -} - inline bool HasHasbit(const FieldDescriptor* descriptor) { // Note that currently message fields inside oneofs have hasbits. This is // surprising, as the oneof case should avoid any need for a hasbit. But if // you change this method to remove hasbits for oneofs, a few tests fail. // TODO(b/124347790): remove hasbits for oneofs return !descriptor->is_repeated() && - (descriptor->has_optional_keyword() || IsProto2(descriptor->file())); + (descriptor->has_optional_keyword() || + descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO2); } // Whether generate classes expose public PARSER instances. @@ -387,12 +378,8 @@ inline bool ExposePublicParser(const FileDescriptor* descriptor) { // Whether unknown enum values are kept (i.e., not stored in UnknownFieldSet // but in the message and can be queried using additional getters that return // ints. -inline bool SupportUnknownEnumValue(const FileDescriptor* descriptor) { - return descriptor->syntax() == FileDescriptor::SYNTAX_PROTO3; -} - inline bool SupportUnknownEnumValue(const FieldDescriptor* field) { - return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3; + return !field->legacy_enum_field_treated_as_closed(); } // Check whether a message has repeated fields. @@ -415,7 +402,7 @@ inline bool IsWrappersProtoFile(const FileDescriptor* descriptor) { } inline bool CheckUtf8(const FieldDescriptor* descriptor) { - return descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || + return descriptor->requires_utf8_validation() || descriptor->file()->options().java_string_check_utf8(); } @@ -448,6 +435,11 @@ int GetExperimentalJavaFieldType(const FieldDescriptor* field); // and the first field number that are not in the table part std::pair GetTableDrivenNumberOfEntriesAndLookUpStartFieldNumber( const FieldDescriptor** fields, int count); + +const FieldDescriptor* MapKeyField(const FieldDescriptor* descriptor); + +const FieldDescriptor* MapValueField(const FieldDescriptor* descriptor); + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/map_field.cc b/src/google/protobuf/compiler/java/map_field.cc index 13354c4ad642..041690b64f8b 100644 --- a/src/google/protobuf/compiler/java/map_field.cc +++ b/src/google/protobuf/compiler/java/map_field.cc @@ -47,20 +47,6 @@ namespace java { namespace { -const FieldDescriptor* KeyField(const FieldDescriptor* descriptor) { - ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); - const Descriptor* message = descriptor->message_type(); - ABSL_CHECK(message->options().map_entry()); - return message->map_key(); -} - -const FieldDescriptor* ValueField(const FieldDescriptor* descriptor) { - ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); - const Descriptor* message = descriptor->message_type(); - ABSL_CHECK(message->options().map_entry()); - return message->map_value(); -} - std::string TypeName(const FieldDescriptor* field, ClassNameResolver* name_resolver, bool boxed) { if (GetJavaType(field) == JAVATYPE_MESSAGE) { @@ -98,8 +84,8 @@ void SetMessageVariables( (*variables)["type"] = name_resolver->GetImmutableClassName(descriptor->message_type()); - const FieldDescriptor* key = KeyField(descriptor); - const FieldDescriptor* value = ValueField(descriptor); + const FieldDescriptor* key = MapKeyField(descriptor); + const FieldDescriptor* value = MapValueField(descriptor); const JavaType keyJavaType = GetJavaType(key); const JavaType valueJavaType = GetJavaType(value); @@ -147,7 +133,7 @@ void SetMessageVariables( {"value_enum_type_pass_through_nullness", absl::StrCat(pass_through_nullness, (*variables)["value_enum_type"])}); - if (SupportUnknownEnumValue(descriptor->file())) { + if (SupportUnknownEnumValue(value)) { // Map unknown values to a special UNRECOGNIZED value if supported. variables->insert( {"unrecognized_value", @@ -244,7 +230,9 @@ void ImmutableMapFieldGenerator::GenerateInterfaceMembers( "$deprecation$boolean ${$contains$capitalized_name$$}$(\n" " $key_type$ key);\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { if (context_->options().opensource_runtime) { printer->Print(variables_, "/**\n" @@ -275,7 +263,7 @@ void ImmutableMapFieldGenerator::GenerateInterfaceMembers( "$deprecation$$value_enum_type$ ${$get$capitalized_name$OrThrow$}$(\n" " $key_type$ key);\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { printer->Print( variables_, "/**\n" @@ -362,7 +350,7 @@ void ImmutableMapFieldGenerator::GenerateMembers(io::Printer* printer) const { " }\n" " return $name$_;\n" "}\n"); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + if (GetJavaType(MapValueField(descriptor_)) == JAVATYPE_ENUM) { printer->Print( variables_, "private static final\n" @@ -434,7 +422,8 @@ void ImmutableMapFieldGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { if (context_->options().opensource_runtime) { printer->Print( variables_, @@ -478,7 +467,7 @@ void ImmutableMapFieldGenerator::GenerateBuilderMembers( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { if (context_->options().opensource_runtime) { printer->Print( variables_, @@ -585,7 +574,8 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( "}\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { if (context_->options().opensource_runtime) { printer->Print( variables_, @@ -645,7 +635,7 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { printer->Print( variables_, "/**\n" @@ -870,8 +860,8 @@ void ImmutableMapFieldGenerator::GenerateBuildingCode( void ImmutableMapFieldGenerator::GenerateBuilderParsingCode( io::Printer* printer) const { - if (!SupportUnknownEnumValue(descriptor_->file()) && - GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + const FieldDescriptor* value = MapValueField(descriptor_); + if (!SupportUnknownEnumValue(value) && GetJavaType(value) == JAVATYPE_ENUM) { printer->Print( variables_, "com.google.protobuf.ByteString bytes = input.readBytes();\n" diff --git a/src/google/protobuf/compiler/java/map_field_lite.cc b/src/google/protobuf/compiler/java/map_field_lite.cc index ca4091a5abcc..8e61a5b14568 100644 --- a/src/google/protobuf/compiler/java/map_field_lite.cc +++ b/src/google/protobuf/compiler/java/map_field_lite.cc @@ -49,20 +49,6 @@ namespace java { namespace { -const FieldDescriptor* KeyField(const FieldDescriptor* descriptor) { - ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); - const Descriptor* message = descriptor->message_type(); - ABSL_CHECK(message->options().map_entry()); - return message->map_key(); -} - -const FieldDescriptor* ValueField(const FieldDescriptor* descriptor) { - ABSL_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); - const Descriptor* message = descriptor->message_type(); - ABSL_CHECK(message->options().map_entry()); - return message->map_value(); -} - std::string TypeName(const FieldDescriptor* field, ClassNameResolver* name_resolver, bool boxed) { if (GetJavaType(field) == JAVATYPE_MESSAGE) { @@ -100,8 +86,8 @@ void SetMessageVariables( ClassNameResolver* name_resolver = context->GetNameResolver(); (*variables)["type"] = name_resolver->GetImmutableClassName(descriptor->message_type()); - const FieldDescriptor* key = KeyField(descriptor); - const FieldDescriptor* value = ValueField(descriptor); + const FieldDescriptor* key = MapKeyField(descriptor); + const FieldDescriptor* value = MapValueField(descriptor); const JavaType keyJavaType = GetJavaType(key); const JavaType valueJavaType = GetJavaType(value); @@ -144,7 +130,7 @@ void SetMessageVariables( {"value_enum_type_pass_through_nullness", absl::StrCat(pass_through_nullness, (*variables)["value_enum_type"])}); - if (SupportUnknownEnumValue(descriptor->file())) { + if (SupportUnknownEnumValue(value)) { // Map unknown values to a special UNRECOGNIZED value if supported. variables->insert( {"unrecognized_value", @@ -217,7 +203,8 @@ void ImmutableMapFieldLiteGenerator::GenerateInterfaceMembers( "$deprecation$boolean ${$contains$capitalized_name$$}$(\n" " $key_type$ key);\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { if (context_->options().opensource_runtime) { printer->Print(variables_, "/**\n" @@ -248,7 +235,7 @@ void ImmutableMapFieldLiteGenerator::GenerateInterfaceMembers( "$deprecation$$value_enum_type$ ${$get$capitalized_name$OrThrow$}$(\n" " $key_type$ key);\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { printer->Print( variables_, "/**\n" @@ -363,7 +350,9 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( " return internalGet$capitalized_name$().containsKey(key);\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { printer->Print( variables_, "private static final\n" @@ -432,7 +421,7 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( " return $name$ValueConverter.doForward(map.get(key));\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { printer->Print( variables_, "/**\n" @@ -545,7 +534,7 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( } // Generate private setters for the builder to proxy into. - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + if (GetJavaType(value) == JAVATYPE_ENUM) { WriteFieldDocComment(printer, descriptor_); printer->Print( variables_, @@ -556,7 +545,7 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( " internalGetMutable$capitalized_name$(),\n" " $name$ValueConverter);\n" "}\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { WriteFieldDocComment(printer, descriptor_); printer->Print( variables_, @@ -583,9 +572,9 @@ void ImmutableMapFieldLiteGenerator::GenerateFieldInfo( printer->Print(variables_, "\"$name$_\",\n" "$default_entry$,\n"); - if (!SupportUnknownEnumValue(descriptor_) && - GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { - PrintEnumVerifierLogic(printer, ValueField(descriptor_), variables_, + const FieldDescriptor* value = MapValueField(descriptor_); + if (!SupportUnknownEnumValue(value) && GetJavaType(value) == JAVATYPE_ENUM) { + PrintEnumVerifierLogic(printer, MapValueField(descriptor_), variables_, /*var_name=*/"$value_enum_type$", /*terminating_string=*/",\n", /*enforce_lite=*/context_->EnforceLite()); @@ -631,7 +620,8 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (GetJavaType(ValueField(descriptor_)) == JAVATYPE_ENUM) { + const FieldDescriptor* value = MapValueField(descriptor_); + if (GetJavaType(value) == JAVATYPE_ENUM) { if (context_->options().opensource_runtime) { printer->Print( variables_, @@ -711,7 +701,7 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); - if (SupportUnknownEnumValue(descriptor_->file())) { + if (SupportUnknownEnumValue(value)) { printer->Print( variables_, "/**\n" diff --git a/src/google/protobuf/compiler/java/message_lite.cc b/src/google/protobuf/compiler/java/message_lite.cc index 721a463c3d9b..b23f0e010170 100644 --- a/src/google/protobuf/compiler/java/message_lite.cc +++ b/src/google/protobuf/compiler/java/message_lite.cc @@ -507,7 +507,7 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodNewBuildMessageInfo( std::vector chars; int flags = 0; - if (IsProto2(descriptor_->file())) { + if (descriptor_->file()->syntax() == FileDescriptor::SYNTAX_PROTO2) { flags |= 0x1; } if (descriptor_->options().message_set_wire_format()) { diff --git a/src/google/protobuf/compiler/java/primitive_field.cc b/src/google/protobuf/compiler/java/primitive_field.cc index 15a1edb46e7a..b30faf6b1a24 100644 --- a/src/google/protobuf/compiler/java/primitive_field.cc +++ b/src/google/protobuf/compiler/java/primitive_field.cc @@ -141,7 +141,7 @@ void SetPrimitiveVariables( } (*variables)["on_changed"] = "onChanged();"; - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); // Note that these have a trailing ";". @@ -221,7 +221,7 @@ int ImmutablePrimitiveFieldGenerator::GetNumBitsForBuilder() const { return 1; } void ImmutablePrimitiveFieldGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean has$capitalized_name$();\n"); @@ -234,7 +234,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateMembers( io::Printer* printer) const { printer->Print(variables_, "private $field_type$ $name$_ = $default$;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -258,7 +258,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { printer->Print(variables_, "private $field_type$ $name$_ $default_init$;\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -332,7 +332,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -363,7 +363,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateBuilderClearCode( void ImmutablePrimitiveFieldGenerator::GenerateMergingCode( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { printer->Print(variables_, "if (other.has$capitalized_name$()) {\n" " set$capitalized_name$(other.get$capitalized_name$());\n" @@ -527,7 +527,7 @@ ImmutablePrimitiveOneofFieldGenerator:: void ImmutablePrimitiveOneofFieldGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -550,7 +550,7 @@ void ImmutablePrimitiveOneofFieldGenerator::GenerateMembers( void ImmutablePrimitiveOneofFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$public boolean ${$has$capitalized_name$$}$() {\n" diff --git a/src/google/protobuf/compiler/java/primitive_field_lite.cc b/src/google/protobuf/compiler/java/primitive_field_lite.cc index b7197b89adc6..95e6034d67ba 100644 --- a/src/google/protobuf/compiler/java/primitive_field_lite.cc +++ b/src/google/protobuf/compiler/java/primitive_field_lite.cc @@ -151,7 +151,7 @@ void SetPrimitiveVariables( (*variables)["fixed_size"] = absl::StrCat(fixed_size); } - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); @@ -219,7 +219,7 @@ int ImmutablePrimitiveFieldLiteGenerator::GetNumBitsForMessage() const { void ImmutablePrimitiveFieldLiteGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean has$capitalized_name$();\n"); @@ -246,7 +246,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( " fieldNumber=$number$,\n" " type=com.google.protobuf.FieldType.$annotation_field_type$,\n" " isRequired=$required$)\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" " presenceBitsId=$bit_field_id$,\n" @@ -255,7 +255,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( } printer->Print(variables_, "private $field_type$ $name$_;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -301,7 +301,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -362,7 +362,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -378,7 +378,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateFieldInfo( WriteIntToUtf16CharSequence(descriptor_->number(), output); WriteIntToUtf16CharSequence(GetExperimentalJavaFieldType(descriptor_), output); - if (HasHasbit(descriptor_)) { + if (descriptor_->has_presence()) { WriteIntToUtf16CharSequence(messageBitIndex_, output); } printer->Print(variables_, "\"$name$_\",\n"); @@ -416,7 +416,7 @@ ImmutablePrimitiveOneofFieldLiteGenerator:: void ImmutablePrimitiveOneofFieldLiteGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -464,7 +464,7 @@ void ImmutablePrimitiveOneofFieldLiteGenerator::GenerateFieldInfo( void ImmutablePrimitiveOneofFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" diff --git a/src/google/protobuf/compiler/java/string_field.cc b/src/google/protobuf/compiler/java/string_field.cc index 0913e0a3ee7a..ce06628181c2 100644 --- a/src/google/protobuf/compiler/java/string_field.cc +++ b/src/google/protobuf/compiler/java/string_field.cc @@ -102,7 +102,7 @@ void SetPrimitiveVariables( : ""}); (*variables)["on_changed"] = "onChanged();"; - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); (*variables)["set_has_field_bit_to_local"] = @@ -197,7 +197,7 @@ int ImmutableStringFieldGenerator::GetNumBitsForBuilder() const { return 1; } // repeated fields, the logic is done in LazyStringArrayList. void ImmutableStringFieldGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean has$capitalized_name$();\n"); @@ -218,7 +218,7 @@ void ImmutableStringFieldGenerator::GenerateMembers( "private volatile java.lang.Object $name$_ = $default$;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -277,7 +277,7 @@ void ImmutableStringFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { printer->Print(variables_, "private java.lang.Object $name$_ $default_init$;\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -395,7 +395,7 @@ void ImmutableStringFieldGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -423,7 +423,7 @@ void ImmutableStringFieldGenerator::GenerateBuilderClearCode( void ImmutableStringFieldGenerator::GenerateMergingCode( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { // Allow a slight breach of abstraction here in order to avoid forcing // all string fields to Strings when copying fields from a Message. printer->Print(variables_, @@ -517,7 +517,7 @@ ImmutableStringOneofFieldGenerator::~ImmutableStringOneofFieldGenerator() {} void ImmutableStringOneofFieldGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$public boolean ${$has$capitalized_name$$}$() {\n" @@ -581,7 +581,7 @@ void ImmutableStringOneofFieldGenerator::GenerateMembers( void ImmutableStringOneofFieldGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" diff --git a/src/google/protobuf/compiler/java/string_field_lite.cc b/src/google/protobuf/compiler/java/string_field_lite.cc index 274df5379056..1e5dc6b610df 100644 --- a/src/google/protobuf/compiler/java/string_field_lite.cc +++ b/src/google/protobuf/compiler/java/string_field_lite.cc @@ -97,7 +97,7 @@ void SetPrimitiveVariables( (*variables)["enforce_utf8"] = CheckUtf8(descriptor) ? "true" : "false"; } - if (HasHasbit(descriptor)) { + if (descriptor->has_presence()) { if (!context->options().opensource_runtime) { (*variables)["bit_field_id"] = absl::StrCat(messageBitIndex / 32); (*variables)["bit_field_name"] = GetBitFieldNameForBit(messageBitIndex); @@ -179,7 +179,7 @@ int ImmutableStringFieldLiteGenerator::GetNumBitsForMessage() const { // shouldn't be necessary or used on devices. void ImmutableStringFieldLiteGenerator::GenerateInterfaceMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "$deprecation$boolean ${$has$capitalized_name$$}$();\n"); @@ -207,7 +207,7 @@ void ImmutableStringFieldLiteGenerator::GenerateMembers( " type=com.google.protobuf.FieldType.$annotation_field_type$,\n" " isRequired=$required$,\n" " isEnforceUtf8=$enforce_utf8$)\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" " presenceBitsId=$bit_field_id$,\n" @@ -217,7 +217,7 @@ void ImmutableStringFieldLiteGenerator::GenerateMembers( printer->Print(variables_, "private java.lang.String $name$_;\n"); PrintExtraFieldInfo(variables_, printer); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -279,7 +279,7 @@ void ImmutableStringFieldLiteGenerator::GenerateMembers( void ImmutableStringFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print( variables_, @@ -361,7 +361,7 @@ void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" "}\n"); - if (HasHazzer(descriptor_)) { + if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, /* builder */ false, /* kdoc */ true); printer->Print( @@ -377,7 +377,7 @@ void ImmutableStringFieldLiteGenerator::GenerateFieldInfo( WriteIntToUtf16CharSequence(descriptor_->number(), output); WriteIntToUtf16CharSequence(GetExperimentalJavaFieldType(descriptor_), output); - if (HasHasbit(descriptor_)) { + if (descriptor_->has_presence()) { WriteIntToUtf16CharSequence(messageBitIndex_, output); } printer->Print(variables_, "\"$name$_\",\n"); @@ -408,7 +408,7 @@ ImmutableStringOneofFieldLiteGenerator:: void ImmutableStringOneofFieldLiteGenerator::GenerateMembers( io::Printer* printer) const { PrintExtraFieldInfo(variables_, printer); - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" @@ -486,7 +486,7 @@ void ImmutableStringOneofFieldLiteGenerator::GenerateFieldInfo( void ImmutableStringOneofFieldLiteGenerator::GenerateBuilderMembers( io::Printer* printer) const { - ABSL_DCHECK(HasHazzer(descriptor_)); + ABSL_DCHECK(descriptor_->has_presence()); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER); printer->Print(variables_, "@java.lang.Override\n" From b6132da7a89e84d17f785efffa3d2368ce4b7e1c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 13:55:36 -0800 Subject: [PATCH 155/463] Create a new, central, OSS-sync-friendly place for our allowlists to live. This will create some mysteriously empty lists in open source; that means they only contain Google-private names. PiperOrigin-RevId: 513915452 --- src/google/protobuf/compiler/BUILD.bazel | 1 + .../protobuf/compiler/allowlists/BUILD.bazel | 42 +++++ .../protobuf/compiler/allowlists/allowlist.h | 148 ++++++++++++++++++ .../compiler/allowlists/allowlist_test.cc | 120 ++++++++++++++ .../compiler/allowlists/allowlists.cc | 53 +++++++ .../protobuf/compiler/allowlists/allowlists.h | 47 ++++++ .../compiler/command_line_interface.cc | 1 + 7 files changed, 412 insertions(+) create mode 100644 src/google/protobuf/compiler/allowlists/BUILD.bazel create mode 100644 src/google/protobuf/compiler/allowlists/allowlist.h create mode 100644 src/google/protobuf/compiler/allowlists/allowlist_test.cc create mode 100644 src/google/protobuf/compiler/allowlists/allowlists.cc create mode 100644 src/google/protobuf/compiler/allowlists/allowlists.h diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index af50b87c604e..89ab5ed60fec 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -90,6 +90,7 @@ cc_library( ":importer", ":retention", "//src/google/protobuf:protobuf_nowkt", + "//src/google/protobuf/compiler/allowlists", "@com_google_absl//absl/container:btree", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", diff --git a/src/google/protobuf/compiler/allowlists/BUILD.bazel b/src/google/protobuf/compiler/allowlists/BUILD.bazel new file mode 100644 index 000000000000..c57091ea08da --- /dev/null +++ b/src/google/protobuf/compiler/allowlists/BUILD.bazel @@ -0,0 +1,42 @@ +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("//build_defs:cpp_opts.bzl", "COPTS") + +package(default_visibility = ["//visibility:private"]) + +cc_library( + name = "allowlist", + hdrs = ["allowlist.h"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/allowlists", + deps = [ + "//src/google/protobuf/stubs", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "allowlists", + srcs = ["allowlists.cc"], + hdrs = ["allowlists.h"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/allowlists", + visibility = ["//src/google/protobuf:__subpackages__"], + deps = [ + ":allowlist", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "allowlist_test", + srcs = ["allowlist_test.cc"], + copts = COPTS, + deps = [ + ":allowlist", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/google/protobuf/compiler/allowlists/allowlist.h b/src/google/protobuf/compiler/allowlists/allowlist.h new file mode 100644 index 000000000000..191c138a4cf7 --- /dev/null +++ b/src/google/protobuf/compiler/allowlists/allowlist.h @@ -0,0 +1,148 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_ALLOWLISTS_ALLOWLIST_H__ +#define GOOGLE_PROTOBUF_COMPILER_ALLOWLISTS_ALLOWLIST_H__ + +#include +#include + +#include "absl/algorithm/container.h" +#include "google/protobuf/stubs/common.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace internal { +enum AllowlistFlags : unsigned int { + kNone = 0, + kMatchPrefix = 1 << 1, + kAllowAllInOss = 1 << 2, +}; + +// An allowlist of things (messages, files, targets) that are allowed to violate +// some constraint. +// +// This is fundamentally a simple API over a set of static strings. It should +// only ever be used as a `static const` variable. +// +// These allowlists are usually only used internally within Google, and contain +// the names of internal files and Protobufs. In open source, these lists become +// no-ops (either they always or never allow everything). +template +class Allowlist final { + public: + template > + constexpr Allowlist(const absl::string_view (&list)[n], AllowlistFlags flags) + : flags_(flags) { + for (size_t i = 0; i < n; ++i) { + list_[i] = list[i]; + if (i != 0) { + ABSL_ASSERT(list_[i - 1] < list_[i] && "Allowlist must be sorted!"); + } + } + } + + template > + explicit constexpr Allowlist(AllowlistFlags flags) + : list_(nullptr, 0), flags_(flags) {} + + // Checks if the element is allowed by this allowlist. + bool Allows(absl::string_view name) const { + if (flags_ & AllowlistFlags::kAllowAllInOss) return true; + + // Convert to a span to get access to standard algorithms without resorting + // to horrible things like std::end(). + absl::Span list = list_; + + auto bound = absl::c_lower_bound(list, name); + if (bound == list.end()) { + // If this string has the last element as a prefix, it will appear as if + // the element is not present in the list; we can take care of this case + // by manually checking the last element. + // + // This will also spuriously fire if a string sorts before everything in + // the list, but in that case the check will still return false as + // expected. + if (flags_ & AllowlistFlags::kMatchPrefix && !list.empty()) { + return absl::StartsWith(name, list.back()); + } + + return false; + } + + if (name == *bound) return true; + + if (flags_ & AllowlistFlags::kMatchPrefix && bound != list.begin()) { + return absl::StartsWith(name, bound[-1]); + } + + return false; + } + + private: + constexpr absl::Span list() const { return list_; } + + // NOTE: std::array::operator[] is *not* constexpr before C++17. + // + // In order for a zero-element list to work, we replace the array with a + // null string view when the size is zero. + std::conditional_t> + list_; + AllowlistFlags flags_; +}; + +struct EmptyAllowlistSentinel {}; + +// This overload picks up MakeAllowlist({}), since zero-length arrays are not +// a thing in C++. +constexpr Allowlist<0> MakeAllowlist( + EmptyAllowlistSentinel, // This binds to `{}`. + AllowlistFlags flags = AllowlistFlags::kNone) { + return Allowlist<0>(flags); +} + +template +constexpr Allowlist MakeAllowlist( + const absl::string_view (&list)[n], + AllowlistFlags flags = AllowlistFlags::kNone) { + return Allowlist(list, flags); +} + +} // namespace internal +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_ALLOWLISTS_ALLOWLIST_H__ diff --git a/src/google/protobuf/compiler/allowlists/allowlist_test.cc b/src/google/protobuf/compiler/allowlists/allowlist_test.cc new file mode 100644 index 000000000000..03dda913b081 --- /dev/null +++ b/src/google/protobuf/compiler/allowlists/allowlist_test.cc @@ -0,0 +1,120 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/allowlists/allowlist.h" + +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace internal { +namespace { + +TEST(AllowlistTest, Smoke) { + static const auto kList = MakeAllowlist({ + "bar", + "baz", + "foo", + }); + + EXPECT_TRUE(kList.Allows("bar")); + EXPECT_TRUE(kList.Allows("baz")); + EXPECT_TRUE(kList.Allows("foo")); + EXPECT_FALSE(kList.Allows("barf")); + EXPECT_FALSE(kList.Allows("baq")); + EXPECT_FALSE(kList.Allows("bak")); + EXPECT_FALSE(kList.Allows("foob")); +} + +TEST(AllowlistTest, Empty) { + static const auto kList = MakeAllowlist({}); + + EXPECT_FALSE(kList.Allows("bar")); + EXPECT_FALSE(kList.Allows("baz")); + EXPECT_FALSE(kList.Allows("foo")); + EXPECT_FALSE(kList.Allows("barf")); + EXPECT_FALSE(kList.Allows("baq")); + EXPECT_FALSE(kList.Allows("bak")); + EXPECT_FALSE(kList.Allows("foob")); +} + +TEST(AllowlistTest, Prefix) { + static const auto kList = MakeAllowlist( + { + "bar", + "baz", + "foo", + }, + AllowlistFlags::kMatchPrefix); + + EXPECT_TRUE(kList.Allows("bar")); + EXPECT_TRUE(kList.Allows("baz")); + EXPECT_TRUE(kList.Allows("foo")); + EXPECT_TRUE(kList.Allows("barf")); + EXPECT_TRUE(kList.Allows("foon")); + EXPECT_TRUE(kList.Allows("bazaar")); + EXPECT_FALSE(kList.Allows("baq")); + EXPECT_FALSE(kList.Allows("bbr")); + EXPECT_FALSE(kList.Allows("fbar")); + EXPECT_FALSE(kList.Allows("ba")); + EXPECT_FALSE(kList.Allows("fon")); + EXPECT_FALSE(kList.Allows("fop")); +} + +TEST(AllowlistTest, Oss) { + static const auto kList = MakeAllowlist( + { + "bar", + "baz", + "foo", + }, + AllowlistFlags::kAllowAllInOss); + + EXPECT_TRUE(kList.Allows("bar")); + EXPECT_TRUE(kList.Allows("baz")); + EXPECT_TRUE(kList.Allows("foo")); + EXPECT_TRUE(kList.Allows("barf")); + EXPECT_TRUE(kList.Allows("baq")); + EXPECT_TRUE(kList.Allows("bak")); + EXPECT_TRUE(kList.Allows("foob")); +} + +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) +TEST(AllowlistTest, Unsorted) { + EXPECT_DEATH(MakeAllowlist({"foo", "bar"}), "Allowlist must be sorted!"); +} +#endif + +} // namespace +} // namespace internal +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/allowlists/allowlists.cc b/src/google/protobuf/compiler/allowlists/allowlists.cc new file mode 100644 index 000000000000..0e4d60f0b6f8 --- /dev/null +++ b/src/google/protobuf/compiler/allowlists/allowlists.cc @@ -0,0 +1,53 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/allowlists/allowlists.h" + +#include "absl/strings/string_view.h" +#include "google/protobuf/compiler/allowlists/allowlist.h" + +namespace google { +namespace protobuf { +namespace compiler { + +// NOTE: Allowlists in this file are not accepting new entries unless otherwise +// specified. + +static constexpr auto kWeakImports = internal::MakeAllowlist({ +// Intentionally left blank. +}); + +bool IsWeakImportFile(absl::string_view filename) { + return kWeakImports.Allows(filename); +} + +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/allowlists/allowlists.h b/src/google/protobuf/compiler/allowlists/allowlists.h new file mode 100644 index 000000000000..b2dc17459e3c --- /dev/null +++ b/src/google/protobuf/compiler/allowlists/allowlists.h @@ -0,0 +1,47 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_ALLOWLISTS_ALLOWLISTS_H__ +#define GOOGLE_PROTOBUF_COMPILER_ALLOWLISTS_ALLOWLISTS_H__ + +#include "absl/strings/string_view.h" + +namespace google { +namespace protobuf { +namespace compiler { + +// Returns whether a file can use the `import weak` syntax. +bool IsWeakImportFile(absl::string_view file); + +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_ALLOWLISTS_COMPILER_ALLOWLISTS_H__ diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index effdd3a66f1e..86ae4b7cadde 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -36,6 +36,7 @@ #include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" +#include "google/protobuf/compiler/allowlists/allowlists.h" #include "google/protobuf/stubs/platform_macros.h" From 72eb4652b77cc74fa3536b6b4f9a0177764f2a95 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 14:24:26 -0800 Subject: [PATCH 156/463] Parallelize the build of the PHP extension PiperOrigin-RevId: 513922943 --- php/tests/compile_extension.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/php/tests/compile_extension.sh b/php/tests/compile_extension.sh index 5f7a6cce69e8..91ed1d47635a 100755 --- a/php/tests/compile_extension.sh +++ b/php/tests/compile_extension.sh @@ -33,6 +33,6 @@ if [[ ! -f BUILD_STAMP ]] || [[ "$(cat BUILD_STAMP)" != "$FINGERPRINT" ]]; then echo "$FINGERPRINT" > BUILD_STAMP fi -make -TEST_PHP_ARGS="-q" make test +make -j8 +TEST_PHP_ARGS="-q" make -j8 test popd > /dev/null From cb405967c3b9939c853c71951990bd186e5ad7b6 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Fri, 3 Mar 2023 14:36:51 -0800 Subject: [PATCH 157/463] Add compiler/cpp/field_generators/cord_field.cc PiperOrigin-RevId: 513926076 --- src/google/protobuf/compiler/cpp/BUILD.bazel | 1 + .../cpp/field_generators/cord_field.cc | 419 ++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100644 src/google/protobuf/compiler/cpp/field_generators/cord_field.cc diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel index bd6944025ea9..e2908b3dc525 100644 --- a/src/google/protobuf/compiler/cpp/BUILD.bazel +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -47,6 +47,7 @@ cc_library( "enum.cc", "extension.cc", "field.cc", + "field_generators/cord_field.cc", "field_generators/enum_field.cc", "field_generators/map_field.cc", "field_generators/message_field.cc", diff --git a/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc b/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc new file mode 100644 index 000000000000..c79c88c0d130 --- /dev/null +++ b/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc @@ -0,0 +1,419 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/substitute.h" +#include "google/protobuf/compiler/cpp/field.h" +#include "google/protobuf/compiler/cpp/field_generators/generators.h" +#include "google/protobuf/compiler/cpp/helpers.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace cpp { +namespace { +void SetCordVariables( + const FieldDescriptor* descriptor, + absl::flat_hash_map* variables, + const Options& options) { + (*variables)["default"] = absl::StrCat( + "\"", absl::CEscape(descriptor->default_value_string()), "\""); + (*variables)["default_length"] = + absl::StrCat(descriptor->default_value_string().length()); + (*variables)["full_name"] = descriptor->full_name(); + // For one of Cords + (*variables)["default_variable_name"] = MakeDefaultName(descriptor); + (*variables)["default_variable_field"] = MakeDefaultFieldName(descriptor); + (*variables)["default_variable"] = + descriptor->default_value_string().empty() + ? ProtobufNamespace(options) + + "::internal::GetEmptyCordAlreadyInited()" + : absl::StrCat( + QualifiedClassName(descriptor->containing_type(), options), + "::", MakeDefaultFieldName(descriptor)); +} + +class CordFieldGenerator : public FieldGeneratorBase { + public: + CordFieldGenerator(const FieldDescriptor* descriptor, const Options& options); + ~CordFieldGenerator() override = default; + + void GeneratePrivateMembers(io::Printer* printer) const override; + void GenerateAccessorDeclarations(io::Printer* printer) const override; + void GenerateInlineAccessorDefinitions(io::Printer* printer) const override; + void GenerateClearingCode(io::Printer* printer) const override; + void GenerateMergingCode(io::Printer* printer) const override; + void GenerateSwappingCode(io::Printer* printer) const override; + void GenerateConstructorCode(io::Printer* printer) const override; + void GenerateDestructorCode(io::Printer* printer) const override; + void GenerateArenaDestructorCode(io::Printer* printer) const override; + void GenerateSerializeWithCachedSizesToArray( + io::Printer* printer) const override; + void GenerateByteSize(io::Printer* printer) const override; + void GenerateAggregateInitializer(io::Printer* printer) const override; + void GenerateConstexprAggregateInitializer( + io::Printer* printer) const override; + ArenaDtorNeeds NeedsArenaDestructor() const override { + return ArenaDtorNeeds::kRequired; + } +}; + +class CordOneofFieldGenerator : public CordFieldGenerator { + public: + CordOneofFieldGenerator(const FieldDescriptor* descriptor, + const Options& options); + ~CordOneofFieldGenerator() override = default; + + void GeneratePrivateMembers(io::Printer* printer) const override; + void GenerateStaticMembers(io::Printer* printer) const override; + void GenerateInlineAccessorDefinitions(io::Printer* printer) const override; + void GenerateNonInlineAccessorDefinitions( + io::Printer* printer) const override; + void GenerateClearingCode(io::Printer* printer) const override; + void GenerateSwappingCode(io::Printer* printer) const override; + void GenerateConstructorCode(io::Printer* printer) const override {} + void GenerateArenaDestructorCode(io::Printer* printer) const override; + // Overrides CordFieldGenerator behavior. + ArenaDtorNeeds NeedsArenaDestructor() const override { + return ArenaDtorNeeds::kNone; + } +}; + + +CordFieldGenerator::CordFieldGenerator(const FieldDescriptor* descriptor, + const Options& options) + : FieldGeneratorBase(descriptor, options) { + SetCordVariables(descriptor, &variables_, options); +} + +void CordFieldGenerator::GeneratePrivateMembers(io::Printer* printer) const { + Formatter format(printer, variables_); + format("::absl::Cord $name$_;\n"); + if (!descriptor_->default_value_string().empty()) { + format( + "struct _default_$name$_func_ {\n" + " constexpr absl::string_view operator()() const {\n" + " return absl::string_view($default$, $default_length$);\n" + " }\n" + "};\n"); + } +} + +void CordFieldGenerator::GenerateAccessorDeclarations( + io::Printer* printer) const { + Formatter format(printer, variables_); + format("$deprecated_attr$const ::absl::Cord& ${1$$name$$}$() const;\n", + descriptor_); + format( + "$deprecated_attr$void ${1$set_$name$$}$(const ::absl::Cord& value);\n" + "$deprecated_attr$void ${1$set_$name$$}$(::absl::string_view value);\n", + std::make_tuple(descriptor_, GeneratedCodeInfo::Annotation::SET)); + format("private:\n"); + format( + "void ${1$_internal_set_$name$$}$(const ::absl::Cord& value);\n" + "::absl::Cord* ${1$_internal_mutable_$name$$}$();\n" + "public:\n", + descriptor_); +} + +void CordFieldGenerator::GenerateInlineAccessorDefinitions( + io::Printer* printer) const { + Formatter format(printer, variables_); + format( + "inline const ::absl::Cord& $classname$::_internal_$name$() const {\n" + " return $field$;\n" + "}\n" + "inline const ::absl::Cord& $classname$::$name$() const {\n" + "$annotate_get$" + " // @@protoc_insertion_point(field_get:$full_name$)\n" + " return _internal_$name$();\n" + "}\n" + "inline void $classname$::_internal_set_$name$(const ::absl::Cord& " + "value) {\n" + " $set_hasbit$\n" + " $field$ = value;\n" + "}\n" + "inline void $classname$::set_$name$(const ::absl::Cord& value) {\n" + "$maybe_prepare_split_message$" + " _internal_set_$name$(value);\n" + "$annotate_set$" + " // @@protoc_insertion_point(field_set:$full_name$)\n" + "}\n" + "inline void $classname$::set_$name$(::absl::string_view value) {\n" + "$maybe_prepare_split_message$" + " $set_hasbit$\n" + " $field$ = value;\n" + "$annotate_set$" + " // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n" + "}\n"); +} + +void CordFieldGenerator::GenerateClearingCode(io::Printer* printer) const { + Formatter format(printer, variables_); + if (descriptor_->default_value_string().empty()) { + format("$field$.clear();\n"); + } else { + format("$field$ = ::absl::string_view($default$, $default_length$);\n"); + } +} + +void CordFieldGenerator::GenerateMergingCode(io::Printer* printer) const { + Formatter format(printer, variables_); + format("_this->_internal_set_$name$(from._internal_$name$());\n"); +} + +void CordFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { + Formatter format(printer, variables_); + format("$field$.swap(other->$field$);\n"); +} + +void CordFieldGenerator::GenerateConstructorCode(io::Printer* printer) const { + ABSL_CHECK(!ShouldSplit(descriptor_, options_)); + Formatter format(printer, variables_); + if (!descriptor_->default_value_string().empty()) { + format("$field$ = ::absl::string_view($default$, $default_length$);\n"); + } +} + +void CordFieldGenerator::GenerateDestructorCode(io::Printer* printer) const { + Formatter format(printer, variables_); + if (ShouldSplit(descriptor_, options_)) { + // A cord field in the `Split` struct is automatically destroyed when the + // split pointer is deleted and should not be explicitly destroyed here. + return; + } + format("$field$.~Cord();\n"); +} + +void CordFieldGenerator::GenerateArenaDestructorCode( + io::Printer* printer) const { + Formatter format(printer, variables_); + // _this is the object being destructed (we are inside a static method here). + format("_this->$field$. ::absl::Cord::~Cord ();\n"); +} + +void CordFieldGenerator::GenerateSerializeWithCachedSizesToArray( + io::Printer* printer) const { + Formatter format(printer, variables_); + if (descriptor_->type() == FieldDescriptor::TYPE_STRING) { + GenerateUtf8CheckCodeForCord( + descriptor_, options_, false, + absl::Substitute("this->_internal_$0(), ", printer->LookupVar("name")), + format); + } + format( + "target = stream->Write$declared_type$($number$, " + "this->_internal_$name$(), " + "target);\n"); +} + +void CordFieldGenerator::GenerateByteSize(io::Printer* printer) const { + Formatter format(printer, variables_); + format( + "total_size += $tag_size$ +\n" + " ::$proto_ns$::internal::WireFormatLite::$declared_type$Size(\n" + " this->_internal_$name$());\n"); +} + +void CordFieldGenerator::GenerateConstexprAggregateInitializer( + io::Printer* printer) const { + Formatter format(printer, variables_); + if (descriptor_->default_value_string().empty()) { + format("/*decltype($field$)*/{}"); + } else { + format( + "/*decltype($field$)*/{::absl::strings_internal::MakeStringConstant(\n" + " $classname$::Impl_::$1$_default_$name$_func_{})}", + ShouldSplit(descriptor_, options_) ? "Split::" : ""); + } +} + +void CordFieldGenerator::GenerateAggregateInitializer( + io::Printer* printer) const { + Formatter format(printer, variables_); + if (ShouldSplit(descriptor_, options_)) { + format("decltype(Impl_::Split::$name$_){}"); + return; + } + format("decltype($field$){}"); +} + +// =================================================================== + +CordOneofFieldGenerator::CordOneofFieldGenerator( + const FieldDescriptor* descriptor, const Options& options) + : CordFieldGenerator(descriptor, options) { +} + +void CordOneofFieldGenerator::GeneratePrivateMembers( + io::Printer* printer) const { + Formatter format(printer, variables_); + format("::absl::Cord *$name$_;\n"); +} + +void CordOneofFieldGenerator::GenerateStaticMembers( + io::Printer* printer) const { + Formatter format(printer, variables_); + if (!descriptor_->default_value_string().empty()) { + format( + "struct _default_$name$_func_ {\n" + " constexpr absl::string_view operator()() const {\n" + " return absl::string_view($default$, $default_length$);\n" + " }\n" + "};" + "static const ::absl::Cord $default_variable_name$;\n"); + } +} + +void CordOneofFieldGenerator::GenerateInlineAccessorDefinitions( + io::Printer* printer) const { + Formatter format(printer, variables_); + format( + "inline const ::absl::Cord& $classname$::_internal_$name$() const {\n" + " if ($has_field$) {\n" + " return *$field$;\n" + " }\n" + " return $default_variable$;\n" + "}\n" + "inline const ::absl::Cord& $classname$::$name$() const {\n" + "$annotate_get$" + " // @@protoc_insertion_point(field_get:$full_name$)\n" + " return _internal_$name$();\n" + "}\n" + "inline void $classname$::_internal_set_$name$(const ::absl::Cord& " + "value) {\n" + " if ($not_has_field$) {\n" + " clear_$oneof_name$();\n" + " set_has_$name$();\n" + " $field$ = new ::absl::Cord;\n" + " if (GetArenaForAllocation() != nullptr) {\n" + " GetArenaForAllocation()->Own($field$);\n" + " }\n" + " }\n" + " *$field$ = value;\n" + "}\n" + "inline void $classname$::set_$name$(const ::absl::Cord& value) {\n" + " _internal_set_$name$(value);\n" + "$annotate_set$" + " // @@protoc_insertion_point(field_set:$full_name$)\n" + "}\n" + "inline void $classname$::set_$name$(::absl::string_view value) {\n" + " if ($not_has_field$) {\n" + " clear_$oneof_name$();\n" + " set_has_$name$();\n" + " $field$ = new ::absl::Cord;\n" + " if (GetArenaForAllocation() != nullptr) {\n" + " GetArenaForAllocation()->Own($field$);\n" + " }\n" + " }\n" + " *$field$ = value;\n" + "$annotate_set$" + " // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n" + "}\n" + "inline ::absl::Cord* $classname$::_internal_mutable_$name$() {\n" + " if ($not_has_field$) {\n" + " clear_$oneof_name$();\n" + " set_has_$name$();\n" + " $field$ = new ::absl::Cord;\n" + " if (GetArenaForAllocation() != nullptr) {\n" + " GetArenaForAllocation()->Own($field$);\n" + " }\n" + " }\n" + " return $field$;\n" + "}\n" + "inline ::absl::Cord* $classname$::mutable_$name$() {\n" + " ::absl::Cord* _cord = _internal_mutable_$name$();\n" + "$annotate_mutable$" + " // @@protoc_insertion_point(field_mutable:$full_name$)\n" + " return _cord;\n" + "}\n"); +} + +void CordOneofFieldGenerator::GenerateNonInlineAccessorDefinitions( + io::Printer* printer) const { + Formatter format(printer, variables_); + if (!descriptor_->default_value_string().empty()) { + format( + "PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT " + "const ::absl::Cord $classname$::$default_variable_field$(\n" + " ::absl::strings_internal::MakeStringConstant(\n" + " _default_$name$_func_{}));\n"); + } +} + +void CordOneofFieldGenerator::GenerateClearingCode(io::Printer* printer) const { + Formatter format(printer, variables_); + format( + "if (GetArenaForAllocation() == nullptr) {\n" + " delete $field$;\n" + "}\n"); +} + +void CordOneofFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { + // Don't print any swapping code. Swapping the union will swap this field. +} + +void CordOneofFieldGenerator::GenerateArenaDestructorCode( + io::Printer* printer) const { + // We inherit from CordFieldGenerator, so we need to re-override to the + // default behavior here. +} + +// =================================================================== +} // namespace + +std::unique_ptr MakeSingularCordGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc) { + return absl::make_unique(desc, options); +} + + +std::unique_ptr MakeOneofCordGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc) { + return absl::make_unique(desc, options); +} + +} // namespace cpp +} // namespace compiler +} // namespace protobuf +} // namespace google From 63895855d7b1298bee97591cbafced49f23902da Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 3 Mar 2023 14:48:13 -0800 Subject: [PATCH 158/463] Internal Code Change PiperOrigin-RevId: 513928860 --- conformance/conformance_test.cc | 17 ++++++++++++----- .../protobuf/compiler/ruby/ruby_generator.cc | 15 +++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index e5d2907ab252..aba07a0f1de4 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -162,12 +162,19 @@ ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const { } string ConformanceTestSuite::ConformanceRequestSetting::GetTestName() const { - string rname = prototype_message_.GetDescriptor()->file()->syntax() == - FileDescriptor::SYNTAX_PROTO3 - ? "Proto3" - : "Proto2"; + string rname; + switch (prototype_message_.GetDescriptor()->file()->syntax()) { + case FileDescriptor::SYNTAX_PROTO3: + rname = ".Proto3."; + break; + case FileDescriptor::SYNTAX_PROTO2: + rname = ".Proto2."; + break; + default: + break; + } - return absl::StrCat(ConformanceLevelToString(level_), ".", rname, ".", + return absl::StrCat(ConformanceLevelToString(level_), rname, InputFormatString(input_format_), ".", test_name_, ".", OutputFormatString(output_format_)); } diff --git a/src/google/protobuf/compiler/ruby/ruby_generator.cc b/src/google/protobuf/compiler/ruby/ruby_generator.cc index fbeb2ef57223..5013fbd5841e 100644 --- a/src/google/protobuf/compiler/ruby/ruby_generator.cc +++ b/src/google/protobuf/compiler/ruby/ruby_generator.cc @@ -79,8 +79,7 @@ std::string GetOutputFilename(absl::string_view proto_file) { } std::string LabelForField(const FieldDescriptor* field) { - if (field->has_optional_keyword() && - field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) { + if (field->has_optional_keyword() && field->containing_oneof() != nullptr) { return "proto3_optional"; } switch (field->label()) { @@ -520,11 +519,9 @@ bool GenerateFile(const FileDescriptor* file, io::Printer* printer, printer->Print("\n"); } - // TODO: Remove this when ruby supports extensions for proto2 syntax. - if (file->syntax() == FileDescriptor::SYNTAX_PROTO2 && - file->extension_count() > 0) { - ABSL_LOG(WARNING) - << "Extensions are not yet supported for proto2 .proto files."; + // TODO: Remove this when ruby supports extensions. + if (file->extension_count() > 0) { + ABSL_LOG(WARNING) << "Extensions are not yet supported in Ruby."; } bool use_raw_descriptor = file->name() == "google/protobuf/descriptor.proto"; @@ -560,9 +557,7 @@ bool Generator::Generate( const std::string& parameter, GeneratorContext* generator_context, std::string* error) const { - - if (file->syntax() != FileDescriptor::SYNTAX_PROTO3 && - file->syntax() != FileDescriptor::SYNTAX_PROTO2) { + if (file->syntax() == FileDescriptor::SYNTAX_UNKNOWN) { *error = "Invalid or unsupported proto syntax"; return false; } From 208f0ad4556bbf32249b6b9b367750f9f31903f8 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Fri, 3 Mar 2023 17:01:33 -0800 Subject: [PATCH 159/463] add hasbit to proto3 fields even when 'optional' is missing PiperOrigin-RevId: 513956259 --- src/google/protobuf/descriptor.cc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index efcd86e35895..9827ac1376a1 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8417,18 +8417,7 @@ bool HasPreservingUnknownEnumSemantics(const FieldDescriptor* field) { } bool HasHasbit(const FieldDescriptor* field) { - // This predicate includes proto3 message fields only if they have "optional". - // Foo submsg1 = 1; // HasHasbit() == false - // optional Foo submsg2 = 2; // HasHasbit() == true - // This is slightly odd, as adding "optional" to a singular proto3 field does - // not change the semantics or API. However whenever any field in a message - // has a hasbit, it forces reflection to include hasbit offsets for *all* - // fields, even if almost all of them are set to -1 (no hasbit). So to avoid - // causing a sudden size regression for ~all proto3 messages, we give proto3 - // message fields a hasbit only if "optional" is present. If the user is - // explicitly writing "optional", it is likely they are writing it on - // primitive fields also. - return (field->has_optional_keyword() || field->is_required()) && + return field->has_presence() && !field->real_containing_oneof() && !field->options().weak(); } From 04a0204bf3916e84f88d324cf5876fbb37ae90bf Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 18:35:49 -0800 Subject: [PATCH 160/463] Internal changes to extensions. PiperOrigin-RevId: 513971799 --- src/google/protobuf/descriptor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index c3be72bd34d2..f1d2032be0dd 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -85,6 +85,7 @@ namespace google { namespace protobuf { + // Defined in this file. class Descriptor; class FieldDescriptor; @@ -118,7 +119,6 @@ class MethodOptions; class FileOptions; class UninterpretedOption; class SourceCodeInfo; -class ExtensionMetadata; // Defined in message.h class Message; From 8f3ae17b3d248456691334de7a3901003acae387 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 Mar 2023 18:44:46 -0800 Subject: [PATCH 161/463] Automated rollback of commit 208f0ad4556bbf32249b6b9b367750f9f31903f8. PiperOrigin-RevId: 513973094 --- src/google/protobuf/descriptor.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 9827ac1376a1..efcd86e35895 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8417,7 +8417,18 @@ bool HasPreservingUnknownEnumSemantics(const FieldDescriptor* field) { } bool HasHasbit(const FieldDescriptor* field) { - return field->has_presence() && !field->real_containing_oneof() && + // This predicate includes proto3 message fields only if they have "optional". + // Foo submsg1 = 1; // HasHasbit() == false + // optional Foo submsg2 = 2; // HasHasbit() == true + // This is slightly odd, as adding "optional" to a singular proto3 field does + // not change the semantics or API. However whenever any field in a message + // has a hasbit, it forces reflection to include hasbit offsets for *all* + // fields, even if almost all of them are set to -1 (no hasbit). So to avoid + // causing a sudden size regression for ~all proto3 messages, we give proto3 + // message fields a hasbit only if "optional" is present. If the user is + // explicitly writing "optional", it is likely they are writing it on + // primitive fields also. + return (field->has_optional_keyword() || field->is_required()) && !field->options().weak(); } From 02bbd4e4924a493e46d18f30922bcd7a8bcca193 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 Mar 2023 15:25:56 +0000 Subject: [PATCH 162/463] Auto-generate files after cl/513973094 --- csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 358616 -> 358586 bytes src/file_lists.cmake | 4 ++++ 2 files changed, 4 insertions(+) diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 66c1815ce162cd19c96898c34a8fa5c290db4547..2dc8612f630d03468aece7c8745683dc576f1aa6 100644 GIT binary patch delta 495 zcmccdQFPZw(S|LI2`gD%)o>Y2|CqxpzP)-SV>eUQ%xf!T86coxZSQ&zyJ6xxFbx)+ z`xV0eF@1yVe;8nFI0E7E|M@6nV;uhl95^zmY7qTT5J{EadVOEOPEf^O+X&EM`}(^K0|nZ zQBKOLJBwubzy}wxYdAqA4$ig|(v)3|ei-4rT;pUL+ z0Fq{4aB!TyYAvG@vdjUf%)#vs)-u{MG5SrH+rTJcW_oEwzbrGyX8vFICde{zaIX40 zK~{{5nd3RDz-@LZW{!hQN=zK7O3WNr7#BU7D$B;9rNopR(cZOz5r~<9n0b5G1{S{) E05g^N+5i9m delta 525 zcmdn>QS`<~(S|LI2`gDTYPpQ2UwqCezP)-SV>eUQg?pQ18D1RQB-;Q4KmG#&)*h;Quil!T@ppGtT|GUX}p}CccBvAoki`FuP&qwH07F zpisl~f;EiF^&tBgLH2?4vV*JwF+iXmWD|(N53&u!00Tylwf}%ju(cot7=Wz>u|U>B zg#P~rSqo&>gRF&$)hp$3i944Tm*nUA7nEd1=I8mBWTX~_CFYc-7Fz{_Trvlyo$>4a z#j?-aT^(QEStR>Avd8JvfzS4%$#R2*7{CG@K>o|= z*=rfKML;GnFgSos00jhtL$bs4RcjfQU@`|mWaQp_9&nLsQSB_@tku#gfn d#}&q3_a*?%;%v{^zzD=lK+L>7X9J7h2>`Q*2Aco? diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 0f933adb267a..3b49e919f6fa 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -272,11 +272,13 @@ set(libprotobuf_lite_hdrs # @//pkg:protoc set(libprotoc_srcs + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/allowlists/allowlists.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/code_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/command_line_interface.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/enum.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/extension.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/field_generators/map_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/field_generators/message_field.cc @@ -370,6 +372,8 @@ set(libprotoc_srcs # @//pkg:protoc set(libprotoc_hdrs + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/allowlists/allowlist.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/allowlists/allowlists.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/code_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/command_line_interface.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/enum.h From 91d77ae291d1f381ba09eff0b6b8f659d1170311 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 6 Mar 2023 08:53:46 -0800 Subject: [PATCH 163/463] Internal changes PiperOrigin-RevId: 514424955 --- .../protobuf/test_messages_proto2.proto | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/src/google/protobuf/test_messages_proto2.proto b/src/google/protobuf/test_messages_proto2.proto index 1cc7c86ae286..088bfa792390 100644 --- a/src/google/protobuf/test_messages_proto2.proto +++ b/src/google/protobuf/test_messages_proto2.proto @@ -301,3 +301,97 @@ message ProtoWithKeywords { optional string concept = 2; repeated string requires = 3; } + +message TestAllRequiredTypesProto2 { + message NestedMessage { + required int32 a = 1; + required TestAllRequiredTypesProto2 corecursive = 2; + optional TestAllRequiredTypesProto2 optional_corecursive = 3; + } + + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + NEG = -1; // Intentionally negative. + } + + // Singular + required int32 required_int32 = 1; + required int64 required_int64 = 2; + required uint32 required_uint32 = 3; + required uint64 required_uint64 = 4; + required sint32 required_sint32 = 5; + required sint64 required_sint64 = 6; + required fixed32 required_fixed32 = 7; + required fixed64 required_fixed64 = 8; + required sfixed32 required_sfixed32 = 9; + required sfixed64 required_sfixed64 = 10; + required float required_float = 11; + required double required_double = 12; + required bool required_bool = 13; + required string required_string = 14; + required bytes required_bytes = 15; + + required NestedMessage required_nested_message = 18; + required ForeignMessageProto2 required_foreign_message = 19; + + required NestedEnum required_nested_enum = 21; + required ForeignEnumProto2 required_foreign_enum = 22; + + required string required_string_piece = 24 [ctype = STRING_PIECE]; + required string required_cord = 25 [ctype = CORD]; + + required TestAllRequiredTypesProto2 recursive_message = 27; + optional TestAllRequiredTypesProto2 optional_recursive_message = 28; + + // extensions + extensions 120 to 200; + + // groups + required group Data = 201 { + required int32 group_int32 = 202; + required uint32 group_uint32 = 203; + } + + // default values + required int32 default_int32 = 241 [default = -123456789]; + required int64 default_int64 = 242 [default = -9123456789123456789]; + required uint32 default_uint32 = 243 [default = 2123456789]; + required uint64 default_uint64 = 244 [default = 10123456789123456789]; + required sint32 default_sint32 = 245 [default = -123456789]; + required sint64 default_sint64 = 246 [default = -9123456789123456789]; + required fixed32 default_fixed32 = 247 [default = 2123456789]; + required fixed64 default_fixed64 = 248 [default = 10123456789123456789]; + required sfixed32 default_sfixed32 = 249 [default = -123456789]; + required sfixed64 default_sfixed64 = 250 [default = -9123456789123456789]; + required float default_float = 251 [default = 9e9]; + required double default_double = 252 [default = 7e22]; + required bool default_bool = 253 [default = true]; + required string default_string = 254 [default = "Rosebud"]; + required bytes default_bytes = 255 [default = "joshua"]; + + // Reserved for unknown fields test. + reserved 1000 to 9999; + + // message_set test case. + message MessageSetCorrect { + option message_set_wire_format = true; + + extensions 4 to max; + } + + message MessageSetCorrectExtension1 { + extend MessageSetCorrect { + optional MessageSetCorrectExtension1 message_set_extension = 1547769; + } + required string str = 25; + } + + message MessageSetCorrectExtension2 { + extend MessageSetCorrect { + optional MessageSetCorrectExtension2 message_set_extension = 4135312; + } + required int32 i = 9; + } +} From f3fba625c67250b5d5f6dc0721c96f303f08e43e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 Mar 2023 17:07:06 +0000 Subject: [PATCH 164/463] Auto-generate files after cl/514424955 --- .../TestMessagesProto2.pb.cs | 3496 ++++++++++++++++- csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 358586 -> 366872 bytes 2 files changed, 3489 insertions(+), 7 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.pb.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.pb.cs index bbfe2e057b3a..d48c50e7567e 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.pb.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.pb.cs @@ -215,12 +215,68 @@ static TestMessagesProto2Reflection() { "cm90bzIiLwoORW51bU9ubHlQcm90bzIiHQoEQm9vbBIKCgZrRmFsc2UQABIJ", "CgVrVHJ1ZRABIh8KD09uZVN0cmluZ1Byb3RvMhIMCgRkYXRhGAEgASgJIkYK", "EVByb3RvV2l0aEtleXdvcmRzEg4KBmlubGluZRgBIAEoBRIPCgdjb25jZXB0", - "GAIgASgJEhAKCHJlcXVpcmVzGAMgAygJKkYKEUZvcmVpZ25FbnVtUHJvdG8y", - "Eg8KC0ZPUkVJR05fRk9PEAASDwoLRk9SRUlHTl9CQVIQARIPCgtGT1JFSUdO", - "X0JBWhACOkoKD2V4dGVuc2lvbl9pbnQzMhIxLnByb3RvYnVmX3Rlc3RfbWVz", - "c2FnZXMucHJvdG8yLlRlc3RBbGxUeXBlc1Byb3RvMhh4IAEoBUI4Cihjb20u", - "Z29vZ2xlLnByb3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8ySAH4AQGiAgZQ", - "cm90bzI=")); + "GAIgASgJEhAKCHJlcXVpcmVzGAMgAygJIt4TChpUZXN0QWxsUmVxdWlyZWRU", + "eXBlc1Byb3RvMhIWCg5yZXF1aXJlZF9pbnQzMhgBIAIoBRIWCg5yZXF1aXJl", + "ZF9pbnQ2NBgCIAIoAxIXCg9yZXF1aXJlZF91aW50MzIYAyACKA0SFwoPcmVx", + "dWlyZWRfdWludDY0GAQgAigEEhcKD3JlcXVpcmVkX3NpbnQzMhgFIAIoERIX", + "Cg9yZXF1aXJlZF9zaW50NjQYBiACKBISGAoQcmVxdWlyZWRfZml4ZWQzMhgH", + "IAIoBxIYChByZXF1aXJlZF9maXhlZDY0GAggAigGEhkKEXJlcXVpcmVkX3Nm", + "aXhlZDMyGAkgAigPEhkKEXJlcXVpcmVkX3NmaXhlZDY0GAogAigQEhYKDnJl", + "cXVpcmVkX2Zsb2F0GAsgAigCEhcKD3JlcXVpcmVkX2RvdWJsZRgMIAIoARIV", + "Cg1yZXF1aXJlZF9ib29sGA0gAigIEhcKD3JlcXVpcmVkX3N0cmluZxgOIAIo", + "CRIWCg5yZXF1aXJlZF9ieXRlcxgPIAIoDBJoChdyZXF1aXJlZF9uZXN0ZWRf", + "bWVzc2FnZRgSIAIoCzJHLnByb3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8y", + "LlRlc3RBbGxSZXF1aXJlZFR5cGVzUHJvdG8yLk5lc3RlZE1lc3NhZ2USVQoY", + "cmVxdWlyZWRfZm9yZWlnbl9tZXNzYWdlGBMgAigLMjMucHJvdG9idWZfdGVz", + "dF9tZXNzYWdlcy5wcm90bzIuRm9yZWlnbk1lc3NhZ2VQcm90bzISYgoUcmVx", + "dWlyZWRfbmVzdGVkX2VudW0YFSACKA4yRC5wcm90b2J1Zl90ZXN0X21lc3Nh", + "Z2VzLnByb3RvMi5UZXN0QWxsUmVxdWlyZWRUeXBlc1Byb3RvMi5OZXN0ZWRF", + "bnVtEk8KFXJlcXVpcmVkX2ZvcmVpZ25fZW51bRgWIAIoDjIwLnByb3RvYnVm", + "X3Rlc3RfbWVzc2FnZXMucHJvdG8yLkZvcmVpZ25FbnVtUHJvdG8yEiEKFXJl", + "cXVpcmVkX3N0cmluZ19waWVjZRgYIAIoCUICCAISGQoNcmVxdWlyZWRfY29y", + "ZBgZIAIoCUICCAESVAoRcmVjdXJzaXZlX21lc3NhZ2UYGyACKAsyOS5wcm90", + "b2J1Zl90ZXN0X21lc3NhZ2VzLnByb3RvMi5UZXN0QWxsUmVxdWlyZWRUeXBl", + "c1Byb3RvMhJdChpvcHRpb25hbF9yZWN1cnNpdmVfbWVzc2FnZRgcIAEoCzI5", + "LnByb3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8yLlRlc3RBbGxSZXF1aXJl", + "ZFR5cGVzUHJvdG8yEk0KBGRhdGEYyQEgAigKMj4ucHJvdG9idWZfdGVzdF9t", + "ZXNzYWdlcy5wcm90bzIuVGVzdEFsbFJlcXVpcmVkVHlwZXNQcm90bzIuRGF0", + "YRIiCg1kZWZhdWx0X2ludDMyGPEBIAIoBToKLTEyMzQ1Njc4ORIsCg1kZWZh", + "dWx0X2ludDY0GPIBIAIoAzoULTkxMjM0NTY3ODkxMjM0NTY3ODkSIwoOZGVm", + "YXVsdF91aW50MzIY8wEgAigNOgoyMTIzNDU2Nzg5Ei0KDmRlZmF1bHRfdWlu", + "dDY0GPQBIAIoBDoUMTAxMjM0NTY3ODkxMjM0NTY3ODkSIwoOZGVmYXVsdF9z", + "aW50MzIY9QEgAigROgotMTIzNDU2Nzg5Ei0KDmRlZmF1bHRfc2ludDY0GPYB", + "IAIoEjoULTkxMjM0NTY3ODkxMjM0NTY3ODkSJAoPZGVmYXVsdF9maXhlZDMy", + "GPcBIAIoBzoKMjEyMzQ1Njc4ORIuCg9kZWZhdWx0X2ZpeGVkNjQY+AEgAigG", + "OhQxMDEyMzQ1Njc4OTEyMzQ1Njc4ORIlChBkZWZhdWx0X3NmaXhlZDMyGPkB", + "IAIoDzoKLTEyMzQ1Njc4ORIvChBkZWZhdWx0X3NmaXhlZDY0GPoBIAIoEDoU", + "LTkxMjM0NTY3ODkxMjM0NTY3ODkSHQoNZGVmYXVsdF9mbG9hdBj7ASACKAI6", + "BTllKzA5Eh4KDmRlZmF1bHRfZG91YmxlGPwBIAIoAToFN2UrMjISGwoMZGVm", + "YXVsdF9ib29sGP0BIAIoCDoEdHJ1ZRIgCg5kZWZhdWx0X3N0cmluZxj+ASAC", + "KAk6B1Jvc2VidWQSHgoNZGVmYXVsdF9ieXRlcxj/ASACKAw6Bmpvc2h1YRrD", + "AQoNTmVzdGVkTWVzc2FnZRIJCgFhGAEgAigFEk4KC2NvcmVjdXJzaXZlGAIg", + "AigLMjkucHJvdG9idWZfdGVzdF9tZXNzYWdlcy5wcm90bzIuVGVzdEFsbFJl", + "cXVpcmVkVHlwZXNQcm90bzISVwoUb3B0aW9uYWxfY29yZWN1cnNpdmUYAyAB", + "KAsyOS5wcm90b2J1Zl90ZXN0X21lc3NhZ2VzLnByb3RvMi5UZXN0QWxsUmVx", + "dWlyZWRUeXBlc1Byb3RvMhozCgREYXRhEhQKC2dyb3VwX2ludDMyGMoBIAIo", + "BRIVCgxncm91cF91aW50MzIYywEgAigNGiEKEU1lc3NhZ2VTZXRDb3JyZWN0", + "KggIBBD/////BzoCCAEa8AEKG01lc3NhZ2VTZXRDb3JyZWN0RXh0ZW5zaW9u", + "MRILCgNzdHIYGSACKAkywwEKFW1lc3NhZ2Vfc2V0X2V4dGVuc2lvbhJLLnBy", + "b3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8yLlRlc3RBbGxSZXF1aXJlZFR5", + "cGVzUHJvdG8yLk1lc3NhZ2VTZXRDb3JyZWN0GPm7XiABKAsyVS5wcm90b2J1", + "Zl90ZXN0X21lc3NhZ2VzLnByb3RvMi5UZXN0QWxsUmVxdWlyZWRUeXBlc1By", + "b3RvMi5NZXNzYWdlU2V0Q29ycmVjdEV4dGVuc2lvbjEa7wEKG01lc3NhZ2VT", + "ZXRDb3JyZWN0RXh0ZW5zaW9uMhIJCgFpGAkgAigFMsQBChVtZXNzYWdlX3Nl", + "dF9leHRlbnNpb24SSy5wcm90b2J1Zl90ZXN0X21lc3NhZ2VzLnByb3RvMi5U", + "ZXN0QWxsUmVxdWlyZWRUeXBlc1Byb3RvMi5NZXNzYWdlU2V0Q29ycmVjdBiQ", + "s/wBIAEoCzJVLnByb3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8yLlRlc3RB", + "bGxSZXF1aXJlZFR5cGVzUHJvdG8yLk1lc3NhZ2VTZXRDb3JyZWN0RXh0ZW5z", + "aW9uMiI5CgpOZXN0ZWRFbnVtEgcKA0ZPTxAAEgcKA0JBUhABEgcKA0JBWhAC", + "EhAKA05FRxD///////////8BKgUIeBDJAUoGCOgHEJBOKkYKEUZvcmVpZ25F", + "bnVtUHJvdG8yEg8KC0ZPUkVJR05fRk9PEAASDwoLRk9SRUlHTl9CQVIQARIP", + "CgtGT1JFSUdOX0JBWhACOkoKD2V4dGVuc2lvbl9pbnQzMhIxLnByb3RvYnVm", + "X3Rlc3RfbWVzc2FnZXMucHJvdG8yLlRlc3RBbGxUeXBlc1Byb3RvMhh4IAEo", + "BUI4Cihjb20uZ29vZ2xlLnByb3RvYnVmX3Rlc3RfbWVzc2FnZXMucHJvdG8y", + "SAH4AQGiAgZQcm90bzI=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ProtobufTestMessages.Proto2.ForeignEnumProto2), }, new pb::Extension[] { TestMessagesProto2Extensions.ExtensionInt32 }, new pbr::GeneratedClrTypeInfo[] { @@ -234,7 +290,12 @@ static TestMessagesProto2Reflection() { new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.NullHypothesisProto2), global::ProtobufTestMessages.Proto2.NullHypothesisProto2.Parser, null, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.EnumOnlyProto2), global::ProtobufTestMessages.Proto2.EnumOnlyProto2.Parser, null, null, new[]{ typeof(global::ProtobufTestMessages.Proto2.EnumOnlyProto2.Types.Bool) }, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.OneStringProto2), global::ProtobufTestMessages.Proto2.OneStringProto2.Parser, new[]{ "Data" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.ProtoWithKeywords), global::ProtobufTestMessages.Proto2.ProtoWithKeywords.Parser, new[]{ "Inline", "Concept", "Requires" }, null, null, null, null) + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.ProtoWithKeywords), global::ProtobufTestMessages.Proto2.ProtoWithKeywords.Parser, new[]{ "Inline", "Concept", "Requires" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Parser, new[]{ "RequiredInt32", "RequiredInt64", "RequiredUint32", "RequiredUint64", "RequiredSint32", "RequiredSint64", "RequiredFixed32", "RequiredFixed64", "RequiredSfixed32", "RequiredSfixed64", "RequiredFloat", "RequiredDouble", "RequiredBool", "RequiredString", "RequiredBytes", "RequiredNestedMessage", "RequiredForeignMessage", "RequiredNestedEnum", "RequiredForeignEnum", "RequiredStringPiece", "RequiredCord", "RecursiveMessage", "OptionalRecursiveMessage", "Data", "DefaultInt32", "DefaultInt64", "DefaultUint32", "DefaultUint64", "DefaultSint32", "DefaultSint64", "DefaultFixed32", "DefaultFixed64", "DefaultSfixed32", "DefaultSfixed64", "DefaultFloat", "DefaultDouble", "DefaultBool", "DefaultString", "DefaultBytes" }, null, new[]{ typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage.Parser, new[]{ "A", "Corecursive", "OptionalCorecursive" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data.Parser, new[]{ "GroupInt32", "GroupUint32" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrect), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrect.Parser, null, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension1), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension1.Parser, new[]{ "Str" }, null, null, new pb::Extension[] { global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension1.Extensions.MessageSetExtension }, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension2), global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension2.Parser, new[]{ "I" }, null, null, new pb::Extension[] { global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension2.Extensions.MessageSetExtension }, null)}) })); } #endregion @@ -8586,6 +8647,3427 @@ public void MergeFrom(pb::CodedInputStream input) { } + public sealed partial class TestAllRequiredTypesProto2 : pb::IExtendableMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new TestAllRequiredTypesProto2()); + private pb::UnknownFieldSet _unknownFields; + private pb::ExtensionSet _extensions; + private pb::ExtensionSet _Extensions { get { return _extensions; } } + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestMessagesProto2Reflection.Descriptor.MessageTypes[7]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestAllRequiredTypesProto2() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestAllRequiredTypesProto2(TestAllRequiredTypesProto2 other) : this() { + _hasBits0 = other._hasBits0; + requiredInt32_ = other.requiredInt32_; + requiredInt64_ = other.requiredInt64_; + requiredUint32_ = other.requiredUint32_; + requiredUint64_ = other.requiredUint64_; + requiredSint32_ = other.requiredSint32_; + requiredSint64_ = other.requiredSint64_; + requiredFixed32_ = other.requiredFixed32_; + requiredFixed64_ = other.requiredFixed64_; + requiredSfixed32_ = other.requiredSfixed32_; + requiredSfixed64_ = other.requiredSfixed64_; + requiredFloat_ = other.requiredFloat_; + requiredDouble_ = other.requiredDouble_; + requiredBool_ = other.requiredBool_; + requiredString_ = other.requiredString_; + requiredBytes_ = other.requiredBytes_; + requiredNestedMessage_ = other.requiredNestedMessage_ != null ? other.requiredNestedMessage_.Clone() : null; + requiredForeignMessage_ = other.requiredForeignMessage_ != null ? other.requiredForeignMessage_.Clone() : null; + requiredNestedEnum_ = other.requiredNestedEnum_; + requiredForeignEnum_ = other.requiredForeignEnum_; + requiredStringPiece_ = other.requiredStringPiece_; + requiredCord_ = other.requiredCord_; + recursiveMessage_ = other.recursiveMessage_ != null ? other.recursiveMessage_.Clone() : null; + optionalRecursiveMessage_ = other.optionalRecursiveMessage_ != null ? other.optionalRecursiveMessage_.Clone() : null; + data_ = other.HasData ? other.data_.Clone() : null; + defaultInt32_ = other.defaultInt32_; + defaultInt64_ = other.defaultInt64_; + defaultUint32_ = other.defaultUint32_; + defaultUint64_ = other.defaultUint64_; + defaultSint32_ = other.defaultSint32_; + defaultSint64_ = other.defaultSint64_; + defaultFixed32_ = other.defaultFixed32_; + defaultFixed64_ = other.defaultFixed64_; + defaultSfixed32_ = other.defaultSfixed32_; + defaultSfixed64_ = other.defaultSfixed64_; + defaultFloat_ = other.defaultFloat_; + defaultDouble_ = other.defaultDouble_; + defaultBool_ = other.defaultBool_; + defaultString_ = other.defaultString_; + defaultBytes_ = other.defaultBytes_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + _extensions = pb::ExtensionSet.Clone(other._extensions); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestAllRequiredTypesProto2 Clone() { + return new TestAllRequiredTypesProto2(this); + } + + ///

Field number for the "required_int32" field. + public const int RequiredInt32FieldNumber = 1; + private readonly static int RequiredInt32DefaultValue = 0; + + private int requiredInt32_; + /// + /// Singular + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int RequiredInt32 { + get { if ((_hasBits0 & 1) != 0) { return requiredInt32_; } else { return RequiredInt32DefaultValue; } } + set { + _hasBits0 |= 1; + requiredInt32_ = value; + } + } + /// Gets whether the "required_int32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredInt32 { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "required_int32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredInt32() { + _hasBits0 &= ~1; + } + + /// Field number for the "required_int64" field. + public const int RequiredInt64FieldNumber = 2; + private readonly static long RequiredInt64DefaultValue = 0L; + + private long requiredInt64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long RequiredInt64 { + get { if ((_hasBits0 & 2) != 0) { return requiredInt64_; } else { return RequiredInt64DefaultValue; } } + set { + _hasBits0 |= 2; + requiredInt64_ = value; + } + } + /// Gets whether the "required_int64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredInt64 { + get { return (_hasBits0 & 2) != 0; } + } + /// Clears the value of the "required_int64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredInt64() { + _hasBits0 &= ~2; + } + + /// Field number for the "required_uint32" field. + public const int RequiredUint32FieldNumber = 3; + private readonly static uint RequiredUint32DefaultValue = 0; + + private uint requiredUint32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint RequiredUint32 { + get { if ((_hasBits0 & 4) != 0) { return requiredUint32_; } else { return RequiredUint32DefaultValue; } } + set { + _hasBits0 |= 4; + requiredUint32_ = value; + } + } + /// Gets whether the "required_uint32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredUint32 { + get { return (_hasBits0 & 4) != 0; } + } + /// Clears the value of the "required_uint32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredUint32() { + _hasBits0 &= ~4; + } + + /// Field number for the "required_uint64" field. + public const int RequiredUint64FieldNumber = 4; + private readonly static ulong RequiredUint64DefaultValue = 0UL; + + private ulong requiredUint64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ulong RequiredUint64 { + get { if ((_hasBits0 & 8) != 0) { return requiredUint64_; } else { return RequiredUint64DefaultValue; } } + set { + _hasBits0 |= 8; + requiredUint64_ = value; + } + } + /// Gets whether the "required_uint64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredUint64 { + get { return (_hasBits0 & 8) != 0; } + } + /// Clears the value of the "required_uint64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredUint64() { + _hasBits0 &= ~8; + } + + /// Field number for the "required_sint32" field. + public const int RequiredSint32FieldNumber = 5; + private readonly static int RequiredSint32DefaultValue = 0; + + private int requiredSint32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int RequiredSint32 { + get { if ((_hasBits0 & 16) != 0) { return requiredSint32_; } else { return RequiredSint32DefaultValue; } } + set { + _hasBits0 |= 16; + requiredSint32_ = value; + } + } + /// Gets whether the "required_sint32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredSint32 { + get { return (_hasBits0 & 16) != 0; } + } + /// Clears the value of the "required_sint32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredSint32() { + _hasBits0 &= ~16; + } + + /// Field number for the "required_sint64" field. + public const int RequiredSint64FieldNumber = 6; + private readonly static long RequiredSint64DefaultValue = 0L; + + private long requiredSint64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long RequiredSint64 { + get { if ((_hasBits0 & 32) != 0) { return requiredSint64_; } else { return RequiredSint64DefaultValue; } } + set { + _hasBits0 |= 32; + requiredSint64_ = value; + } + } + /// Gets whether the "required_sint64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredSint64 { + get { return (_hasBits0 & 32) != 0; } + } + /// Clears the value of the "required_sint64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredSint64() { + _hasBits0 &= ~32; + } + + /// Field number for the "required_fixed32" field. + public const int RequiredFixed32FieldNumber = 7; + private readonly static uint RequiredFixed32DefaultValue = 0; + + private uint requiredFixed32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint RequiredFixed32 { + get { if ((_hasBits0 & 64) != 0) { return requiredFixed32_; } else { return RequiredFixed32DefaultValue; } } + set { + _hasBits0 |= 64; + requiredFixed32_ = value; + } + } + /// Gets whether the "required_fixed32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredFixed32 { + get { return (_hasBits0 & 64) != 0; } + } + /// Clears the value of the "required_fixed32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredFixed32() { + _hasBits0 &= ~64; + } + + /// Field number for the "required_fixed64" field. + public const int RequiredFixed64FieldNumber = 8; + private readonly static ulong RequiredFixed64DefaultValue = 0UL; + + private ulong requiredFixed64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ulong RequiredFixed64 { + get { if ((_hasBits0 & 128) != 0) { return requiredFixed64_; } else { return RequiredFixed64DefaultValue; } } + set { + _hasBits0 |= 128; + requiredFixed64_ = value; + } + } + /// Gets whether the "required_fixed64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredFixed64 { + get { return (_hasBits0 & 128) != 0; } + } + /// Clears the value of the "required_fixed64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredFixed64() { + _hasBits0 &= ~128; + } + + /// Field number for the "required_sfixed32" field. + public const int RequiredSfixed32FieldNumber = 9; + private readonly static int RequiredSfixed32DefaultValue = 0; + + private int requiredSfixed32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int RequiredSfixed32 { + get { if ((_hasBits0 & 256) != 0) { return requiredSfixed32_; } else { return RequiredSfixed32DefaultValue; } } + set { + _hasBits0 |= 256; + requiredSfixed32_ = value; + } + } + /// Gets whether the "required_sfixed32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredSfixed32 { + get { return (_hasBits0 & 256) != 0; } + } + /// Clears the value of the "required_sfixed32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredSfixed32() { + _hasBits0 &= ~256; + } + + /// Field number for the "required_sfixed64" field. + public const int RequiredSfixed64FieldNumber = 10; + private readonly static long RequiredSfixed64DefaultValue = 0L; + + private long requiredSfixed64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long RequiredSfixed64 { + get { if ((_hasBits0 & 512) != 0) { return requiredSfixed64_; } else { return RequiredSfixed64DefaultValue; } } + set { + _hasBits0 |= 512; + requiredSfixed64_ = value; + } + } + /// Gets whether the "required_sfixed64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredSfixed64 { + get { return (_hasBits0 & 512) != 0; } + } + /// Clears the value of the "required_sfixed64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredSfixed64() { + _hasBits0 &= ~512; + } + + /// Field number for the "required_float" field. + public const int RequiredFloatFieldNumber = 11; + private readonly static float RequiredFloatDefaultValue = 0F; + + private float requiredFloat_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public float RequiredFloat { + get { if ((_hasBits0 & 1024) != 0) { return requiredFloat_; } else { return RequiredFloatDefaultValue; } } + set { + _hasBits0 |= 1024; + requiredFloat_ = value; + } + } + /// Gets whether the "required_float" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredFloat { + get { return (_hasBits0 & 1024) != 0; } + } + /// Clears the value of the "required_float" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredFloat() { + _hasBits0 &= ~1024; + } + + /// Field number for the "required_double" field. + public const int RequiredDoubleFieldNumber = 12; + private readonly static double RequiredDoubleDefaultValue = 0D; + + private double requiredDouble_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public double RequiredDouble { + get { if ((_hasBits0 & 2048) != 0) { return requiredDouble_; } else { return RequiredDoubleDefaultValue; } } + set { + _hasBits0 |= 2048; + requiredDouble_ = value; + } + } + /// Gets whether the "required_double" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredDouble { + get { return (_hasBits0 & 2048) != 0; } + } + /// Clears the value of the "required_double" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredDouble() { + _hasBits0 &= ~2048; + } + + /// Field number for the "required_bool" field. + public const int RequiredBoolFieldNumber = 13; + private readonly static bool RequiredBoolDefaultValue = false; + + private bool requiredBool_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool RequiredBool { + get { if ((_hasBits0 & 4096) != 0) { return requiredBool_; } else { return RequiredBoolDefaultValue; } } + set { + _hasBits0 |= 4096; + requiredBool_ = value; + } + } + /// Gets whether the "required_bool" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredBool { + get { return (_hasBits0 & 4096) != 0; } + } + /// Clears the value of the "required_bool" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredBool() { + _hasBits0 &= ~4096; + } + + /// Field number for the "required_string" field. + public const int RequiredStringFieldNumber = 14; + private readonly static string RequiredStringDefaultValue = ""; + + private string requiredString_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string RequiredString { + get { return requiredString_ ?? RequiredStringDefaultValue; } + set { + requiredString_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "required_string" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredString { + get { return requiredString_ != null; } + } + /// Clears the value of the "required_string" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredString() { + requiredString_ = null; + } + + /// Field number for the "required_bytes" field. + public const int RequiredBytesFieldNumber = 15; + private readonly static pb::ByteString RequiredBytesDefaultValue = pb::ByteString.Empty; + + private pb::ByteString requiredBytes_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString RequiredBytes { + get { return requiredBytes_ ?? RequiredBytesDefaultValue; } + set { + requiredBytes_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "required_bytes" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredBytes { + get { return requiredBytes_ != null; } + } + /// Clears the value of the "required_bytes" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredBytes() { + requiredBytes_ = null; + } + + /// Field number for the "required_nested_message" field. + public const int RequiredNestedMessageFieldNumber = 18; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage requiredNestedMessage_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage RequiredNestedMessage { + get { return requiredNestedMessage_; } + set { + requiredNestedMessage_ = value; + } + } + + /// Field number for the "required_foreign_message" field. + public const int RequiredForeignMessageFieldNumber = 19; + private global::ProtobufTestMessages.Proto2.ForeignMessageProto2 requiredForeignMessage_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.ForeignMessageProto2 RequiredForeignMessage { + get { return requiredForeignMessage_; } + set { + requiredForeignMessage_ = value; + } + } + + /// Field number for the "required_nested_enum" field. + public const int RequiredNestedEnumFieldNumber = 21; + private readonly static global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum RequiredNestedEnumDefaultValue = global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum.Foo; + + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum requiredNestedEnum_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum RequiredNestedEnum { + get { if ((_hasBits0 & 8192) != 0) { return requiredNestedEnum_; } else { return RequiredNestedEnumDefaultValue; } } + set { + _hasBits0 |= 8192; + requiredNestedEnum_ = value; + } + } + /// Gets whether the "required_nested_enum" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredNestedEnum { + get { return (_hasBits0 & 8192) != 0; } + } + /// Clears the value of the "required_nested_enum" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredNestedEnum() { + _hasBits0 &= ~8192; + } + + /// Field number for the "required_foreign_enum" field. + public const int RequiredForeignEnumFieldNumber = 22; + private readonly static global::ProtobufTestMessages.Proto2.ForeignEnumProto2 RequiredForeignEnumDefaultValue = global::ProtobufTestMessages.Proto2.ForeignEnumProto2.ForeignFoo; + + private global::ProtobufTestMessages.Proto2.ForeignEnumProto2 requiredForeignEnum_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.ForeignEnumProto2 RequiredForeignEnum { + get { if ((_hasBits0 & 16384) != 0) { return requiredForeignEnum_; } else { return RequiredForeignEnumDefaultValue; } } + set { + _hasBits0 |= 16384; + requiredForeignEnum_ = value; + } + } + /// Gets whether the "required_foreign_enum" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredForeignEnum { + get { return (_hasBits0 & 16384) != 0; } + } + /// Clears the value of the "required_foreign_enum" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredForeignEnum() { + _hasBits0 &= ~16384; + } + + /// Field number for the "required_string_piece" field. + public const int RequiredStringPieceFieldNumber = 24; + private readonly static string RequiredStringPieceDefaultValue = ""; + + private string requiredStringPiece_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string RequiredStringPiece { + get { return requiredStringPiece_ ?? RequiredStringPieceDefaultValue; } + set { + requiredStringPiece_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "required_string_piece" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredStringPiece { + get { return requiredStringPiece_ != null; } + } + /// Clears the value of the "required_string_piece" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredStringPiece() { + requiredStringPiece_ = null; + } + + /// Field number for the "required_cord" field. + public const int RequiredCordFieldNumber = 25; + private readonly static string RequiredCordDefaultValue = ""; + + private string requiredCord_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string RequiredCord { + get { return requiredCord_ ?? RequiredCordDefaultValue; } + set { + requiredCord_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "required_cord" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequiredCord { + get { return requiredCord_ != null; } + } + /// Clears the value of the "required_cord" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequiredCord() { + requiredCord_ = null; + } + + /// Field number for the "recursive_message" field. + public const int RecursiveMessageFieldNumber = 27; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 recursiveMessage_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 RecursiveMessage { + get { return recursiveMessage_; } + set { + recursiveMessage_ = value; + } + } + + /// Field number for the "optional_recursive_message" field. + public const int OptionalRecursiveMessageFieldNumber = 28; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 optionalRecursiveMessage_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 OptionalRecursiveMessage { + get { return optionalRecursiveMessage_; } + set { + optionalRecursiveMessage_ = value; + } + } + + /// Field number for the "data" field. + public const int DataFieldNumber = 201; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data data_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data Data { + get { return data_; } + set { + data_ = value; + } + } + /// Gets whether the data field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasData { + get { return data_ != null; } + } + /// Clears the value of the data field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearData() { + data_ = null; + } + + /// Field number for the "default_int32" field. + public const int DefaultInt32FieldNumber = 241; + private readonly static int DefaultInt32DefaultValue = -123456789; + + private int defaultInt32_; + /// + /// default values + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int DefaultInt32 { + get { if ((_hasBits0 & 32768) != 0) { return defaultInt32_; } else { return DefaultInt32DefaultValue; } } + set { + _hasBits0 |= 32768; + defaultInt32_ = value; + } + } + /// Gets whether the "default_int32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultInt32 { + get { return (_hasBits0 & 32768) != 0; } + } + /// Clears the value of the "default_int32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultInt32() { + _hasBits0 &= ~32768; + } + + /// Field number for the "default_int64" field. + public const int DefaultInt64FieldNumber = 242; + private readonly static long DefaultInt64DefaultValue = -9123456789123456789L; + + private long defaultInt64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long DefaultInt64 { + get { if ((_hasBits0 & 65536) != 0) { return defaultInt64_; } else { return DefaultInt64DefaultValue; } } + set { + _hasBits0 |= 65536; + defaultInt64_ = value; + } + } + /// Gets whether the "default_int64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultInt64 { + get { return (_hasBits0 & 65536) != 0; } + } + /// Clears the value of the "default_int64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultInt64() { + _hasBits0 &= ~65536; + } + + /// Field number for the "default_uint32" field. + public const int DefaultUint32FieldNumber = 243; + private readonly static uint DefaultUint32DefaultValue = 2123456789; + + private uint defaultUint32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint DefaultUint32 { + get { if ((_hasBits0 & 131072) != 0) { return defaultUint32_; } else { return DefaultUint32DefaultValue; } } + set { + _hasBits0 |= 131072; + defaultUint32_ = value; + } + } + /// Gets whether the "default_uint32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultUint32 { + get { return (_hasBits0 & 131072) != 0; } + } + /// Clears the value of the "default_uint32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultUint32() { + _hasBits0 &= ~131072; + } + + /// Field number for the "default_uint64" field. + public const int DefaultUint64FieldNumber = 244; + private readonly static ulong DefaultUint64DefaultValue = 10123456789123456789UL; + + private ulong defaultUint64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ulong DefaultUint64 { + get { if ((_hasBits0 & 262144) != 0) { return defaultUint64_; } else { return DefaultUint64DefaultValue; } } + set { + _hasBits0 |= 262144; + defaultUint64_ = value; + } + } + /// Gets whether the "default_uint64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultUint64 { + get { return (_hasBits0 & 262144) != 0; } + } + /// Clears the value of the "default_uint64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultUint64() { + _hasBits0 &= ~262144; + } + + /// Field number for the "default_sint32" field. + public const int DefaultSint32FieldNumber = 245; + private readonly static int DefaultSint32DefaultValue = -123456789; + + private int defaultSint32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int DefaultSint32 { + get { if ((_hasBits0 & 524288) != 0) { return defaultSint32_; } else { return DefaultSint32DefaultValue; } } + set { + _hasBits0 |= 524288; + defaultSint32_ = value; + } + } + /// Gets whether the "default_sint32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultSint32 { + get { return (_hasBits0 & 524288) != 0; } + } + /// Clears the value of the "default_sint32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultSint32() { + _hasBits0 &= ~524288; + } + + /// Field number for the "default_sint64" field. + public const int DefaultSint64FieldNumber = 246; + private readonly static long DefaultSint64DefaultValue = -9123456789123456789L; + + private long defaultSint64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long DefaultSint64 { + get { if ((_hasBits0 & 1048576) != 0) { return defaultSint64_; } else { return DefaultSint64DefaultValue; } } + set { + _hasBits0 |= 1048576; + defaultSint64_ = value; + } + } + /// Gets whether the "default_sint64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultSint64 { + get { return (_hasBits0 & 1048576) != 0; } + } + /// Clears the value of the "default_sint64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultSint64() { + _hasBits0 &= ~1048576; + } + + /// Field number for the "default_fixed32" field. + public const int DefaultFixed32FieldNumber = 247; + private readonly static uint DefaultFixed32DefaultValue = 2123456789; + + private uint defaultFixed32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint DefaultFixed32 { + get { if ((_hasBits0 & 2097152) != 0) { return defaultFixed32_; } else { return DefaultFixed32DefaultValue; } } + set { + _hasBits0 |= 2097152; + defaultFixed32_ = value; + } + } + /// Gets whether the "default_fixed32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultFixed32 { + get { return (_hasBits0 & 2097152) != 0; } + } + /// Clears the value of the "default_fixed32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultFixed32() { + _hasBits0 &= ~2097152; + } + + /// Field number for the "default_fixed64" field. + public const int DefaultFixed64FieldNumber = 248; + private readonly static ulong DefaultFixed64DefaultValue = 10123456789123456789UL; + + private ulong defaultFixed64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ulong DefaultFixed64 { + get { if ((_hasBits0 & 4194304) != 0) { return defaultFixed64_; } else { return DefaultFixed64DefaultValue; } } + set { + _hasBits0 |= 4194304; + defaultFixed64_ = value; + } + } + /// Gets whether the "default_fixed64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultFixed64 { + get { return (_hasBits0 & 4194304) != 0; } + } + /// Clears the value of the "default_fixed64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultFixed64() { + _hasBits0 &= ~4194304; + } + + /// Field number for the "default_sfixed32" field. + public const int DefaultSfixed32FieldNumber = 249; + private readonly static int DefaultSfixed32DefaultValue = -123456789; + + private int defaultSfixed32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int DefaultSfixed32 { + get { if ((_hasBits0 & 8388608) != 0) { return defaultSfixed32_; } else { return DefaultSfixed32DefaultValue; } } + set { + _hasBits0 |= 8388608; + defaultSfixed32_ = value; + } + } + /// Gets whether the "default_sfixed32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultSfixed32 { + get { return (_hasBits0 & 8388608) != 0; } + } + /// Clears the value of the "default_sfixed32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultSfixed32() { + _hasBits0 &= ~8388608; + } + + /// Field number for the "default_sfixed64" field. + public const int DefaultSfixed64FieldNumber = 250; + private readonly static long DefaultSfixed64DefaultValue = -9123456789123456789L; + + private long defaultSfixed64_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public long DefaultSfixed64 { + get { if ((_hasBits0 & 16777216) != 0) { return defaultSfixed64_; } else { return DefaultSfixed64DefaultValue; } } + set { + _hasBits0 |= 16777216; + defaultSfixed64_ = value; + } + } + /// Gets whether the "default_sfixed64" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultSfixed64 { + get { return (_hasBits0 & 16777216) != 0; } + } + /// Clears the value of the "default_sfixed64" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultSfixed64() { + _hasBits0 &= ~16777216; + } + + /// Field number for the "default_float" field. + public const int DefaultFloatFieldNumber = 251; + private readonly static float DefaultFloatDefaultValue = 9e+09F; + + private float defaultFloat_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public float DefaultFloat { + get { if ((_hasBits0 & 33554432) != 0) { return defaultFloat_; } else { return DefaultFloatDefaultValue; } } + set { + _hasBits0 |= 33554432; + defaultFloat_ = value; + } + } + /// Gets whether the "default_float" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultFloat { + get { return (_hasBits0 & 33554432) != 0; } + } + /// Clears the value of the "default_float" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultFloat() { + _hasBits0 &= ~33554432; + } + + /// Field number for the "default_double" field. + public const int DefaultDoubleFieldNumber = 252; + private readonly static double DefaultDoubleDefaultValue = 7e+22D; + + private double defaultDouble_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public double DefaultDouble { + get { if ((_hasBits0 & 67108864) != 0) { return defaultDouble_; } else { return DefaultDoubleDefaultValue; } } + set { + _hasBits0 |= 67108864; + defaultDouble_ = value; + } + } + /// Gets whether the "default_double" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultDouble { + get { return (_hasBits0 & 67108864) != 0; } + } + /// Clears the value of the "default_double" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultDouble() { + _hasBits0 &= ~67108864; + } + + /// Field number for the "default_bool" field. + public const int DefaultBoolFieldNumber = 253; + private readonly static bool DefaultBoolDefaultValue = true; + + private bool defaultBool_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool DefaultBool { + get { if ((_hasBits0 & 134217728) != 0) { return defaultBool_; } else { return DefaultBoolDefaultValue; } } + set { + _hasBits0 |= 134217728; + defaultBool_ = value; + } + } + /// Gets whether the "default_bool" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultBool { + get { return (_hasBits0 & 134217728) != 0; } + } + /// Clears the value of the "default_bool" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultBool() { + _hasBits0 &= ~134217728; + } + + /// Field number for the "default_string" field. + public const int DefaultStringFieldNumber = 254; + private readonly static string DefaultStringDefaultValue = global::System.Text.Encoding.UTF8.GetString(global::System.Convert.FromBase64String("Um9zZWJ1ZA=="), 0, 7); + + private string defaultString_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string DefaultString { + get { return defaultString_ ?? DefaultStringDefaultValue; } + set { + defaultString_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "default_string" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultString { + get { return defaultString_ != null; } + } + /// Clears the value of the "default_string" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultString() { + defaultString_ = null; + } + + /// Field number for the "default_bytes" field. + public const int DefaultBytesFieldNumber = 255; + private readonly static pb::ByteString DefaultBytesDefaultValue = pb::ByteString.FromBase64("am9zaHVh"); + + private pb::ByteString defaultBytes_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString DefaultBytes { + get { return defaultBytes_ ?? DefaultBytesDefaultValue; } + set { + defaultBytes_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "default_bytes" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasDefaultBytes { + get { return defaultBytes_ != null; } + } + /// Clears the value of the "default_bytes" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearDefaultBytes() { + defaultBytes_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as TestAllRequiredTypesProto2); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(TestAllRequiredTypesProto2 other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (RequiredInt32 != other.RequiredInt32) return false; + if (RequiredInt64 != other.RequiredInt64) return false; + if (RequiredUint32 != other.RequiredUint32) return false; + if (RequiredUint64 != other.RequiredUint64) return false; + if (RequiredSint32 != other.RequiredSint32) return false; + if (RequiredSint64 != other.RequiredSint64) return false; + if (RequiredFixed32 != other.RequiredFixed32) return false; + if (RequiredFixed64 != other.RequiredFixed64) return false; + if (RequiredSfixed32 != other.RequiredSfixed32) return false; + if (RequiredSfixed64 != other.RequiredSfixed64) return false; + if (!pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.Equals(RequiredFloat, other.RequiredFloat)) return false; + if (!pbc::ProtobufEqualityComparers.BitwiseDoubleEqualityComparer.Equals(RequiredDouble, other.RequiredDouble)) return false; + if (RequiredBool != other.RequiredBool) return false; + if (RequiredString != other.RequiredString) return false; + if (RequiredBytes != other.RequiredBytes) return false; + if (!object.Equals(RequiredNestedMessage, other.RequiredNestedMessage)) return false; + if (!object.Equals(RequiredForeignMessage, other.RequiredForeignMessage)) return false; + if (RequiredNestedEnum != other.RequiredNestedEnum) return false; + if (RequiredForeignEnum != other.RequiredForeignEnum) return false; + if (RequiredStringPiece != other.RequiredStringPiece) return false; + if (RequiredCord != other.RequiredCord) return false; + if (!object.Equals(RecursiveMessage, other.RecursiveMessage)) return false; + if (!object.Equals(OptionalRecursiveMessage, other.OptionalRecursiveMessage)) return false; + if (!object.Equals(Data, other.Data)) return false; + if (DefaultInt32 != other.DefaultInt32) return false; + if (DefaultInt64 != other.DefaultInt64) return false; + if (DefaultUint32 != other.DefaultUint32) return false; + if (DefaultUint64 != other.DefaultUint64) return false; + if (DefaultSint32 != other.DefaultSint32) return false; + if (DefaultSint64 != other.DefaultSint64) return false; + if (DefaultFixed32 != other.DefaultFixed32) return false; + if (DefaultFixed64 != other.DefaultFixed64) return false; + if (DefaultSfixed32 != other.DefaultSfixed32) return false; + if (DefaultSfixed64 != other.DefaultSfixed64) return false; + if (!pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.Equals(DefaultFloat, other.DefaultFloat)) return false; + if (!pbc::ProtobufEqualityComparers.BitwiseDoubleEqualityComparer.Equals(DefaultDouble, other.DefaultDouble)) return false; + if (DefaultBool != other.DefaultBool) return false; + if (DefaultString != other.DefaultString) return false; + if (DefaultBytes != other.DefaultBytes) return false; + if (!Equals(_extensions, other._extensions)) { + return false; + } + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasRequiredInt32) hash ^= RequiredInt32.GetHashCode(); + if (HasRequiredInt64) hash ^= RequiredInt64.GetHashCode(); + if (HasRequiredUint32) hash ^= RequiredUint32.GetHashCode(); + if (HasRequiredUint64) hash ^= RequiredUint64.GetHashCode(); + if (HasRequiredSint32) hash ^= RequiredSint32.GetHashCode(); + if (HasRequiredSint64) hash ^= RequiredSint64.GetHashCode(); + if (HasRequiredFixed32) hash ^= RequiredFixed32.GetHashCode(); + if (HasRequiredFixed64) hash ^= RequiredFixed64.GetHashCode(); + if (HasRequiredSfixed32) hash ^= RequiredSfixed32.GetHashCode(); + if (HasRequiredSfixed64) hash ^= RequiredSfixed64.GetHashCode(); + if (HasRequiredFloat) hash ^= pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.GetHashCode(RequiredFloat); + if (HasRequiredDouble) hash ^= pbc::ProtobufEqualityComparers.BitwiseDoubleEqualityComparer.GetHashCode(RequiredDouble); + if (HasRequiredBool) hash ^= RequiredBool.GetHashCode(); + if (HasRequiredString) hash ^= RequiredString.GetHashCode(); + if (HasRequiredBytes) hash ^= RequiredBytes.GetHashCode(); + if (requiredNestedMessage_ != null) hash ^= RequiredNestedMessage.GetHashCode(); + if (requiredForeignMessage_ != null) hash ^= RequiredForeignMessage.GetHashCode(); + if (HasRequiredNestedEnum) hash ^= RequiredNestedEnum.GetHashCode(); + if (HasRequiredForeignEnum) hash ^= RequiredForeignEnum.GetHashCode(); + if (HasRequiredStringPiece) hash ^= RequiredStringPiece.GetHashCode(); + if (HasRequiredCord) hash ^= RequiredCord.GetHashCode(); + if (recursiveMessage_ != null) hash ^= RecursiveMessage.GetHashCode(); + if (optionalRecursiveMessage_ != null) hash ^= OptionalRecursiveMessage.GetHashCode(); + if (HasData) hash ^= Data.GetHashCode(); + if (HasDefaultInt32) hash ^= DefaultInt32.GetHashCode(); + if (HasDefaultInt64) hash ^= DefaultInt64.GetHashCode(); + if (HasDefaultUint32) hash ^= DefaultUint32.GetHashCode(); + if (HasDefaultUint64) hash ^= DefaultUint64.GetHashCode(); + if (HasDefaultSint32) hash ^= DefaultSint32.GetHashCode(); + if (HasDefaultSint64) hash ^= DefaultSint64.GetHashCode(); + if (HasDefaultFixed32) hash ^= DefaultFixed32.GetHashCode(); + if (HasDefaultFixed64) hash ^= DefaultFixed64.GetHashCode(); + if (HasDefaultSfixed32) hash ^= DefaultSfixed32.GetHashCode(); + if (HasDefaultSfixed64) hash ^= DefaultSfixed64.GetHashCode(); + if (HasDefaultFloat) hash ^= pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.GetHashCode(DefaultFloat); + if (HasDefaultDouble) hash ^= pbc::ProtobufEqualityComparers.BitwiseDoubleEqualityComparer.GetHashCode(DefaultDouble); + if (HasDefaultBool) hash ^= DefaultBool.GetHashCode(); + if (HasDefaultString) hash ^= DefaultString.GetHashCode(); + if (HasDefaultBytes) hash ^= DefaultBytes.GetHashCode(); + if (_extensions != null) { + hash ^= _extensions.GetHashCode(); + } + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasRequiredInt32) { + output.WriteRawTag(8); + output.WriteInt32(RequiredInt32); + } + if (HasRequiredInt64) { + output.WriteRawTag(16); + output.WriteInt64(RequiredInt64); + } + if (HasRequiredUint32) { + output.WriteRawTag(24); + output.WriteUInt32(RequiredUint32); + } + if (HasRequiredUint64) { + output.WriteRawTag(32); + output.WriteUInt64(RequiredUint64); + } + if (HasRequiredSint32) { + output.WriteRawTag(40); + output.WriteSInt32(RequiredSint32); + } + if (HasRequiredSint64) { + output.WriteRawTag(48); + output.WriteSInt64(RequiredSint64); + } + if (HasRequiredFixed32) { + output.WriteRawTag(61); + output.WriteFixed32(RequiredFixed32); + } + if (HasRequiredFixed64) { + output.WriteRawTag(65); + output.WriteFixed64(RequiredFixed64); + } + if (HasRequiredSfixed32) { + output.WriteRawTag(77); + output.WriteSFixed32(RequiredSfixed32); + } + if (HasRequiredSfixed64) { + output.WriteRawTag(81); + output.WriteSFixed64(RequiredSfixed64); + } + if (HasRequiredFloat) { + output.WriteRawTag(93); + output.WriteFloat(RequiredFloat); + } + if (HasRequiredDouble) { + output.WriteRawTag(97); + output.WriteDouble(RequiredDouble); + } + if (HasRequiredBool) { + output.WriteRawTag(104); + output.WriteBool(RequiredBool); + } + if (HasRequiredString) { + output.WriteRawTag(114); + output.WriteString(RequiredString); + } + if (HasRequiredBytes) { + output.WriteRawTag(122); + output.WriteBytes(RequiredBytes); + } + if (requiredNestedMessage_ != null) { + output.WriteRawTag(146, 1); + output.WriteMessage(RequiredNestedMessage); + } + if (requiredForeignMessage_ != null) { + output.WriteRawTag(154, 1); + output.WriteMessage(RequiredForeignMessage); + } + if (HasRequiredNestedEnum) { + output.WriteRawTag(168, 1); + output.WriteEnum((int) RequiredNestedEnum); + } + if (HasRequiredForeignEnum) { + output.WriteRawTag(176, 1); + output.WriteEnum((int) RequiredForeignEnum); + } + if (HasRequiredStringPiece) { + output.WriteRawTag(194, 1); + output.WriteString(RequiredStringPiece); + } + if (HasRequiredCord) { + output.WriteRawTag(202, 1); + output.WriteString(RequiredCord); + } + if (recursiveMessage_ != null) { + output.WriteRawTag(218, 1); + output.WriteMessage(RecursiveMessage); + } + if (optionalRecursiveMessage_ != null) { + output.WriteRawTag(226, 1); + output.WriteMessage(OptionalRecursiveMessage); + } + if (HasData) { + output.WriteRawTag(203, 12); + output.WriteGroup(Data); + output.WriteRawTag(204, 12); + } + if (HasDefaultInt32) { + output.WriteRawTag(136, 15); + output.WriteInt32(DefaultInt32); + } + if (HasDefaultInt64) { + output.WriteRawTag(144, 15); + output.WriteInt64(DefaultInt64); + } + if (HasDefaultUint32) { + output.WriteRawTag(152, 15); + output.WriteUInt32(DefaultUint32); + } + if (HasDefaultUint64) { + output.WriteRawTag(160, 15); + output.WriteUInt64(DefaultUint64); + } + if (HasDefaultSint32) { + output.WriteRawTag(168, 15); + output.WriteSInt32(DefaultSint32); + } + if (HasDefaultSint64) { + output.WriteRawTag(176, 15); + output.WriteSInt64(DefaultSint64); + } + if (HasDefaultFixed32) { + output.WriteRawTag(189, 15); + output.WriteFixed32(DefaultFixed32); + } + if (HasDefaultFixed64) { + output.WriteRawTag(193, 15); + output.WriteFixed64(DefaultFixed64); + } + if (HasDefaultSfixed32) { + output.WriteRawTag(205, 15); + output.WriteSFixed32(DefaultSfixed32); + } + if (HasDefaultSfixed64) { + output.WriteRawTag(209, 15); + output.WriteSFixed64(DefaultSfixed64); + } + if (HasDefaultFloat) { + output.WriteRawTag(221, 15); + output.WriteFloat(DefaultFloat); + } + if (HasDefaultDouble) { + output.WriteRawTag(225, 15); + output.WriteDouble(DefaultDouble); + } + if (HasDefaultBool) { + output.WriteRawTag(232, 15); + output.WriteBool(DefaultBool); + } + if (HasDefaultString) { + output.WriteRawTag(242, 15); + output.WriteString(DefaultString); + } + if (HasDefaultBytes) { + output.WriteRawTag(250, 15); + output.WriteBytes(DefaultBytes); + } + if (_extensions != null) { + _extensions.WriteTo(output); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasRequiredInt32) { + output.WriteRawTag(8); + output.WriteInt32(RequiredInt32); + } + if (HasRequiredInt64) { + output.WriteRawTag(16); + output.WriteInt64(RequiredInt64); + } + if (HasRequiredUint32) { + output.WriteRawTag(24); + output.WriteUInt32(RequiredUint32); + } + if (HasRequiredUint64) { + output.WriteRawTag(32); + output.WriteUInt64(RequiredUint64); + } + if (HasRequiredSint32) { + output.WriteRawTag(40); + output.WriteSInt32(RequiredSint32); + } + if (HasRequiredSint64) { + output.WriteRawTag(48); + output.WriteSInt64(RequiredSint64); + } + if (HasRequiredFixed32) { + output.WriteRawTag(61); + output.WriteFixed32(RequiredFixed32); + } + if (HasRequiredFixed64) { + output.WriteRawTag(65); + output.WriteFixed64(RequiredFixed64); + } + if (HasRequiredSfixed32) { + output.WriteRawTag(77); + output.WriteSFixed32(RequiredSfixed32); + } + if (HasRequiredSfixed64) { + output.WriteRawTag(81); + output.WriteSFixed64(RequiredSfixed64); + } + if (HasRequiredFloat) { + output.WriteRawTag(93); + output.WriteFloat(RequiredFloat); + } + if (HasRequiredDouble) { + output.WriteRawTag(97); + output.WriteDouble(RequiredDouble); + } + if (HasRequiredBool) { + output.WriteRawTag(104); + output.WriteBool(RequiredBool); + } + if (HasRequiredString) { + output.WriteRawTag(114); + output.WriteString(RequiredString); + } + if (HasRequiredBytes) { + output.WriteRawTag(122); + output.WriteBytes(RequiredBytes); + } + if (requiredNestedMessage_ != null) { + output.WriteRawTag(146, 1); + output.WriteMessage(RequiredNestedMessage); + } + if (requiredForeignMessage_ != null) { + output.WriteRawTag(154, 1); + output.WriteMessage(RequiredForeignMessage); + } + if (HasRequiredNestedEnum) { + output.WriteRawTag(168, 1); + output.WriteEnum((int) RequiredNestedEnum); + } + if (HasRequiredForeignEnum) { + output.WriteRawTag(176, 1); + output.WriteEnum((int) RequiredForeignEnum); + } + if (HasRequiredStringPiece) { + output.WriteRawTag(194, 1); + output.WriteString(RequiredStringPiece); + } + if (HasRequiredCord) { + output.WriteRawTag(202, 1); + output.WriteString(RequiredCord); + } + if (recursiveMessage_ != null) { + output.WriteRawTag(218, 1); + output.WriteMessage(RecursiveMessage); + } + if (optionalRecursiveMessage_ != null) { + output.WriteRawTag(226, 1); + output.WriteMessage(OptionalRecursiveMessage); + } + if (HasData) { + output.WriteRawTag(203, 12); + output.WriteGroup(Data); + output.WriteRawTag(204, 12); + } + if (HasDefaultInt32) { + output.WriteRawTag(136, 15); + output.WriteInt32(DefaultInt32); + } + if (HasDefaultInt64) { + output.WriteRawTag(144, 15); + output.WriteInt64(DefaultInt64); + } + if (HasDefaultUint32) { + output.WriteRawTag(152, 15); + output.WriteUInt32(DefaultUint32); + } + if (HasDefaultUint64) { + output.WriteRawTag(160, 15); + output.WriteUInt64(DefaultUint64); + } + if (HasDefaultSint32) { + output.WriteRawTag(168, 15); + output.WriteSInt32(DefaultSint32); + } + if (HasDefaultSint64) { + output.WriteRawTag(176, 15); + output.WriteSInt64(DefaultSint64); + } + if (HasDefaultFixed32) { + output.WriteRawTag(189, 15); + output.WriteFixed32(DefaultFixed32); + } + if (HasDefaultFixed64) { + output.WriteRawTag(193, 15); + output.WriteFixed64(DefaultFixed64); + } + if (HasDefaultSfixed32) { + output.WriteRawTag(205, 15); + output.WriteSFixed32(DefaultSfixed32); + } + if (HasDefaultSfixed64) { + output.WriteRawTag(209, 15); + output.WriteSFixed64(DefaultSfixed64); + } + if (HasDefaultFloat) { + output.WriteRawTag(221, 15); + output.WriteFloat(DefaultFloat); + } + if (HasDefaultDouble) { + output.WriteRawTag(225, 15); + output.WriteDouble(DefaultDouble); + } + if (HasDefaultBool) { + output.WriteRawTag(232, 15); + output.WriteBool(DefaultBool); + } + if (HasDefaultString) { + output.WriteRawTag(242, 15); + output.WriteString(DefaultString); + } + if (HasDefaultBytes) { + output.WriteRawTag(250, 15); + output.WriteBytes(DefaultBytes); + } + if (_extensions != null) { + _extensions.WriteTo(ref output); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasRequiredInt32) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(RequiredInt32); + } + if (HasRequiredInt64) { + size += 1 + pb::CodedOutputStream.ComputeInt64Size(RequiredInt64); + } + if (HasRequiredUint32) { + size += 1 + pb::CodedOutputStream.ComputeUInt32Size(RequiredUint32); + } + if (HasRequiredUint64) { + size += 1 + pb::CodedOutputStream.ComputeUInt64Size(RequiredUint64); + } + if (HasRequiredSint32) { + size += 1 + pb::CodedOutputStream.ComputeSInt32Size(RequiredSint32); + } + if (HasRequiredSint64) { + size += 1 + pb::CodedOutputStream.ComputeSInt64Size(RequiredSint64); + } + if (HasRequiredFixed32) { + size += 1 + 4; + } + if (HasRequiredFixed64) { + size += 1 + 8; + } + if (HasRequiredSfixed32) { + size += 1 + 4; + } + if (HasRequiredSfixed64) { + size += 1 + 8; + } + if (HasRequiredFloat) { + size += 1 + 4; + } + if (HasRequiredDouble) { + size += 1 + 8; + } + if (HasRequiredBool) { + size += 1 + 1; + } + if (HasRequiredString) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(RequiredString); + } + if (HasRequiredBytes) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(RequiredBytes); + } + if (requiredNestedMessage_ != null) { + size += 2 + pb::CodedOutputStream.ComputeMessageSize(RequiredNestedMessage); + } + if (requiredForeignMessage_ != null) { + size += 2 + pb::CodedOutputStream.ComputeMessageSize(RequiredForeignMessage); + } + if (HasRequiredNestedEnum) { + size += 2 + pb::CodedOutputStream.ComputeEnumSize((int) RequiredNestedEnum); + } + if (HasRequiredForeignEnum) { + size += 2 + pb::CodedOutputStream.ComputeEnumSize((int) RequiredForeignEnum); + } + if (HasRequiredStringPiece) { + size += 2 + pb::CodedOutputStream.ComputeStringSize(RequiredStringPiece); + } + if (HasRequiredCord) { + size += 2 + pb::CodedOutputStream.ComputeStringSize(RequiredCord); + } + if (recursiveMessage_ != null) { + size += 2 + pb::CodedOutputStream.ComputeMessageSize(RecursiveMessage); + } + if (optionalRecursiveMessage_ != null) { + size += 2 + pb::CodedOutputStream.ComputeMessageSize(OptionalRecursiveMessage); + } + if (HasData) { + size += 4 + pb::CodedOutputStream.ComputeGroupSize(Data); + } + if (HasDefaultInt32) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(DefaultInt32); + } + if (HasDefaultInt64) { + size += 2 + pb::CodedOutputStream.ComputeInt64Size(DefaultInt64); + } + if (HasDefaultUint32) { + size += 2 + pb::CodedOutputStream.ComputeUInt32Size(DefaultUint32); + } + if (HasDefaultUint64) { + size += 2 + pb::CodedOutputStream.ComputeUInt64Size(DefaultUint64); + } + if (HasDefaultSint32) { + size += 2 + pb::CodedOutputStream.ComputeSInt32Size(DefaultSint32); + } + if (HasDefaultSint64) { + size += 2 + pb::CodedOutputStream.ComputeSInt64Size(DefaultSint64); + } + if (HasDefaultFixed32) { + size += 2 + 4; + } + if (HasDefaultFixed64) { + size += 2 + 8; + } + if (HasDefaultSfixed32) { + size += 2 + 4; + } + if (HasDefaultSfixed64) { + size += 2 + 8; + } + if (HasDefaultFloat) { + size += 2 + 4; + } + if (HasDefaultDouble) { + size += 2 + 8; + } + if (HasDefaultBool) { + size += 2 + 1; + } + if (HasDefaultString) { + size += 2 + pb::CodedOutputStream.ComputeStringSize(DefaultString); + } + if (HasDefaultBytes) { + size += 2 + pb::CodedOutputStream.ComputeBytesSize(DefaultBytes); + } + if (_extensions != null) { + size += _extensions.CalculateSize(); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(TestAllRequiredTypesProto2 other) { + if (other == null) { + return; + } + if (other.HasRequiredInt32) { + RequiredInt32 = other.RequiredInt32; + } + if (other.HasRequiredInt64) { + RequiredInt64 = other.RequiredInt64; + } + if (other.HasRequiredUint32) { + RequiredUint32 = other.RequiredUint32; + } + if (other.HasRequiredUint64) { + RequiredUint64 = other.RequiredUint64; + } + if (other.HasRequiredSint32) { + RequiredSint32 = other.RequiredSint32; + } + if (other.HasRequiredSint64) { + RequiredSint64 = other.RequiredSint64; + } + if (other.HasRequiredFixed32) { + RequiredFixed32 = other.RequiredFixed32; + } + if (other.HasRequiredFixed64) { + RequiredFixed64 = other.RequiredFixed64; + } + if (other.HasRequiredSfixed32) { + RequiredSfixed32 = other.RequiredSfixed32; + } + if (other.HasRequiredSfixed64) { + RequiredSfixed64 = other.RequiredSfixed64; + } + if (other.HasRequiredFloat) { + RequiredFloat = other.RequiredFloat; + } + if (other.HasRequiredDouble) { + RequiredDouble = other.RequiredDouble; + } + if (other.HasRequiredBool) { + RequiredBool = other.RequiredBool; + } + if (other.HasRequiredString) { + RequiredString = other.RequiredString; + } + if (other.HasRequiredBytes) { + RequiredBytes = other.RequiredBytes; + } + if (other.requiredNestedMessage_ != null) { + if (requiredNestedMessage_ == null) { + RequiredNestedMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage(); + } + RequiredNestedMessage.MergeFrom(other.RequiredNestedMessage); + } + if (other.requiredForeignMessage_ != null) { + if (requiredForeignMessage_ == null) { + RequiredForeignMessage = new global::ProtobufTestMessages.Proto2.ForeignMessageProto2(); + } + RequiredForeignMessage.MergeFrom(other.RequiredForeignMessage); + } + if (other.HasRequiredNestedEnum) { + RequiredNestedEnum = other.RequiredNestedEnum; + } + if (other.HasRequiredForeignEnum) { + RequiredForeignEnum = other.RequiredForeignEnum; + } + if (other.HasRequiredStringPiece) { + RequiredStringPiece = other.RequiredStringPiece; + } + if (other.HasRequiredCord) { + RequiredCord = other.RequiredCord; + } + if (other.recursiveMessage_ != null) { + if (recursiveMessage_ == null) { + RecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + RecursiveMessage.MergeFrom(other.RecursiveMessage); + } + if (other.optionalRecursiveMessage_ != null) { + if (optionalRecursiveMessage_ == null) { + OptionalRecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + OptionalRecursiveMessage.MergeFrom(other.OptionalRecursiveMessage); + } + if (other.HasData) { + if (!HasData) { + Data = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data(); + } + Data.MergeFrom(other.Data); + } + if (other.HasDefaultInt32) { + DefaultInt32 = other.DefaultInt32; + } + if (other.HasDefaultInt64) { + DefaultInt64 = other.DefaultInt64; + } + if (other.HasDefaultUint32) { + DefaultUint32 = other.DefaultUint32; + } + if (other.HasDefaultUint64) { + DefaultUint64 = other.DefaultUint64; + } + if (other.HasDefaultSint32) { + DefaultSint32 = other.DefaultSint32; + } + if (other.HasDefaultSint64) { + DefaultSint64 = other.DefaultSint64; + } + if (other.HasDefaultFixed32) { + DefaultFixed32 = other.DefaultFixed32; + } + if (other.HasDefaultFixed64) { + DefaultFixed64 = other.DefaultFixed64; + } + if (other.HasDefaultSfixed32) { + DefaultSfixed32 = other.DefaultSfixed32; + } + if (other.HasDefaultSfixed64) { + DefaultSfixed64 = other.DefaultSfixed64; + } + if (other.HasDefaultFloat) { + DefaultFloat = other.DefaultFloat; + } + if (other.HasDefaultDouble) { + DefaultDouble = other.DefaultDouble; + } + if (other.HasDefaultBool) { + DefaultBool = other.DefaultBool; + } + if (other.HasDefaultString) { + DefaultString = other.DefaultString; + } + if (other.HasDefaultBytes) { + DefaultBytes = other.DefaultBytes; + } + pb::ExtensionSet.MergeFrom(ref _extensions, other._extensions); + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + if (!pb::ExtensionSet.TryMergeFieldFrom(ref _extensions, input)) { + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + } + break; + case 8: { + RequiredInt32 = input.ReadInt32(); + break; + } + case 16: { + RequiredInt64 = input.ReadInt64(); + break; + } + case 24: { + RequiredUint32 = input.ReadUInt32(); + break; + } + case 32: { + RequiredUint64 = input.ReadUInt64(); + break; + } + case 40: { + RequiredSint32 = input.ReadSInt32(); + break; + } + case 48: { + RequiredSint64 = input.ReadSInt64(); + break; + } + case 61: { + RequiredFixed32 = input.ReadFixed32(); + break; + } + case 65: { + RequiredFixed64 = input.ReadFixed64(); + break; + } + case 77: { + RequiredSfixed32 = input.ReadSFixed32(); + break; + } + case 81: { + RequiredSfixed64 = input.ReadSFixed64(); + break; + } + case 93: { + RequiredFloat = input.ReadFloat(); + break; + } + case 97: { + RequiredDouble = input.ReadDouble(); + break; + } + case 104: { + RequiredBool = input.ReadBool(); + break; + } + case 114: { + RequiredString = input.ReadString(); + break; + } + case 122: { + RequiredBytes = input.ReadBytes(); + break; + } + case 146: { + if (requiredNestedMessage_ == null) { + RequiredNestedMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage(); + } + input.ReadMessage(RequiredNestedMessage); + break; + } + case 154: { + if (requiredForeignMessage_ == null) { + RequiredForeignMessage = new global::ProtobufTestMessages.Proto2.ForeignMessageProto2(); + } + input.ReadMessage(RequiredForeignMessage); + break; + } + case 168: { + RequiredNestedEnum = (global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum) input.ReadEnum(); + break; + } + case 176: { + RequiredForeignEnum = (global::ProtobufTestMessages.Proto2.ForeignEnumProto2) input.ReadEnum(); + break; + } + case 194: { + RequiredStringPiece = input.ReadString(); + break; + } + case 202: { + RequiredCord = input.ReadString(); + break; + } + case 218: { + if (recursiveMessage_ == null) { + RecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(RecursiveMessage); + break; + } + case 226: { + if (optionalRecursiveMessage_ == null) { + OptionalRecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(OptionalRecursiveMessage); + break; + } + case 1611: { + if (!HasData) { + Data = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data(); + } + input.ReadGroup(Data); + break; + } + case 1928: { + DefaultInt32 = input.ReadInt32(); + break; + } + case 1936: { + DefaultInt64 = input.ReadInt64(); + break; + } + case 1944: { + DefaultUint32 = input.ReadUInt32(); + break; + } + case 1952: { + DefaultUint64 = input.ReadUInt64(); + break; + } + case 1960: { + DefaultSint32 = input.ReadSInt32(); + break; + } + case 1968: { + DefaultSint64 = input.ReadSInt64(); + break; + } + case 1981: { + DefaultFixed32 = input.ReadFixed32(); + break; + } + case 1985: { + DefaultFixed64 = input.ReadFixed64(); + break; + } + case 1997: { + DefaultSfixed32 = input.ReadSFixed32(); + break; + } + case 2001: { + DefaultSfixed64 = input.ReadSFixed64(); + break; + } + case 2013: { + DefaultFloat = input.ReadFloat(); + break; + } + case 2017: { + DefaultDouble = input.ReadDouble(); + break; + } + case 2024: { + DefaultBool = input.ReadBool(); + break; + } + case 2034: { + DefaultString = input.ReadString(); + break; + } + case 2042: { + DefaultBytes = input.ReadBytes(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + if (!pb::ExtensionSet.TryMergeFieldFrom(ref _extensions, ref input)) { + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + } + break; + case 8: { + RequiredInt32 = input.ReadInt32(); + break; + } + case 16: { + RequiredInt64 = input.ReadInt64(); + break; + } + case 24: { + RequiredUint32 = input.ReadUInt32(); + break; + } + case 32: { + RequiredUint64 = input.ReadUInt64(); + break; + } + case 40: { + RequiredSint32 = input.ReadSInt32(); + break; + } + case 48: { + RequiredSint64 = input.ReadSInt64(); + break; + } + case 61: { + RequiredFixed32 = input.ReadFixed32(); + break; + } + case 65: { + RequiredFixed64 = input.ReadFixed64(); + break; + } + case 77: { + RequiredSfixed32 = input.ReadSFixed32(); + break; + } + case 81: { + RequiredSfixed64 = input.ReadSFixed64(); + break; + } + case 93: { + RequiredFloat = input.ReadFloat(); + break; + } + case 97: { + RequiredDouble = input.ReadDouble(); + break; + } + case 104: { + RequiredBool = input.ReadBool(); + break; + } + case 114: { + RequiredString = input.ReadString(); + break; + } + case 122: { + RequiredBytes = input.ReadBytes(); + break; + } + case 146: { + if (requiredNestedMessage_ == null) { + RequiredNestedMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedMessage(); + } + input.ReadMessage(RequiredNestedMessage); + break; + } + case 154: { + if (requiredForeignMessage_ == null) { + RequiredForeignMessage = new global::ProtobufTestMessages.Proto2.ForeignMessageProto2(); + } + input.ReadMessage(RequiredForeignMessage); + break; + } + case 168: { + RequiredNestedEnum = (global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.NestedEnum) input.ReadEnum(); + break; + } + case 176: { + RequiredForeignEnum = (global::ProtobufTestMessages.Proto2.ForeignEnumProto2) input.ReadEnum(); + break; + } + case 194: { + RequiredStringPiece = input.ReadString(); + break; + } + case 202: { + RequiredCord = input.ReadString(); + break; + } + case 218: { + if (recursiveMessage_ == null) { + RecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(RecursiveMessage); + break; + } + case 226: { + if (optionalRecursiveMessage_ == null) { + OptionalRecursiveMessage = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(OptionalRecursiveMessage); + break; + } + case 1611: { + if (!HasData) { + Data = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.Data(); + } + input.ReadGroup(Data); + break; + } + case 1928: { + DefaultInt32 = input.ReadInt32(); + break; + } + case 1936: { + DefaultInt64 = input.ReadInt64(); + break; + } + case 1944: { + DefaultUint32 = input.ReadUInt32(); + break; + } + case 1952: { + DefaultUint64 = input.ReadUInt64(); + break; + } + case 1960: { + DefaultSint32 = input.ReadSInt32(); + break; + } + case 1968: { + DefaultSint64 = input.ReadSInt64(); + break; + } + case 1981: { + DefaultFixed32 = input.ReadFixed32(); + break; + } + case 1985: { + DefaultFixed64 = input.ReadFixed64(); + break; + } + case 1997: { + DefaultSfixed32 = input.ReadSFixed32(); + break; + } + case 2001: { + DefaultSfixed64 = input.ReadSFixed64(); + break; + } + case 2013: { + DefaultFloat = input.ReadFloat(); + break; + } + case 2017: { + DefaultDouble = input.ReadDouble(); + break; + } + case 2024: { + DefaultBool = input.ReadBool(); + break; + } + case 2034: { + DefaultString = input.ReadString(); + break; + } + case 2042: { + DefaultBytes = input.ReadBytes(); + break; + } + } + } + } + #endif + + public TValue GetExtension(pb::Extension extension) { + return pb::ExtensionSet.Get(ref _extensions, extension); + } + public pbc::RepeatedField GetExtension(pb::RepeatedExtension extension) { + return pb::ExtensionSet.Get(ref _extensions, extension); + } + public pbc::RepeatedField GetOrInitializeExtension(pb::RepeatedExtension extension) { + return pb::ExtensionSet.GetOrInitialize(ref _extensions, extension); + } + public void SetExtension(pb::Extension extension, TValue value) { + pb::ExtensionSet.Set(ref _extensions, extension, value); + } + public bool HasExtension(pb::Extension extension) { + return pb::ExtensionSet.Has(ref _extensions, extension); + } + public void ClearExtension(pb::Extension extension) { + pb::ExtensionSet.Clear(ref _extensions, extension); + } + public void ClearExtension(pb::RepeatedExtension extension) { + pb::ExtensionSet.Clear(ref _extensions, extension); + } + + #region Nested types + /// Container for nested types declared in the TestAllRequiredTypesProto2 message type. + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static partial class Types { + public enum NestedEnum { + [pbr::OriginalName("FOO")] Foo = 0, + [pbr::OriginalName("BAR")] Bar = 1, + [pbr::OriginalName("BAZ")] Baz = 2, + /// + /// Intentionally negative. + /// + [pbr::OriginalName("NEG")] Neg = -1, + } + + public sealed partial class NestedMessage : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new NestedMessage()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Descriptor.NestedTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public NestedMessage() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public NestedMessage(NestedMessage other) : this() { + _hasBits0 = other._hasBits0; + a_ = other.a_; + corecursive_ = other.corecursive_ != null ? other.corecursive_.Clone() : null; + optionalCorecursive_ = other.optionalCorecursive_ != null ? other.optionalCorecursive_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public NestedMessage Clone() { + return new NestedMessage(this); + } + + /// Field number for the "a" field. + public const int AFieldNumber = 1; + private readonly static int ADefaultValue = 0; + + private int a_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int A { + get { if ((_hasBits0 & 1) != 0) { return a_; } else { return ADefaultValue; } } + set { + _hasBits0 |= 1; + a_ = value; + } + } + /// Gets whether the "a" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasA { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "a" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearA() { + _hasBits0 &= ~1; + } + + /// Field number for the "corecursive" field. + public const int CorecursiveFieldNumber = 2; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 corecursive_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 Corecursive { + get { return corecursive_; } + set { + corecursive_ = value; + } + } + + /// Field number for the "optional_corecursive" field. + public const int OptionalCorecursiveFieldNumber = 3; + private global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 optionalCorecursive_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2 OptionalCorecursive { + get { return optionalCorecursive_; } + set { + optionalCorecursive_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as NestedMessage); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(NestedMessage other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (A != other.A) return false; + if (!object.Equals(Corecursive, other.Corecursive)) return false; + if (!object.Equals(OptionalCorecursive, other.OptionalCorecursive)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasA) hash ^= A.GetHashCode(); + if (corecursive_ != null) hash ^= Corecursive.GetHashCode(); + if (optionalCorecursive_ != null) hash ^= OptionalCorecursive.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasA) { + output.WriteRawTag(8); + output.WriteInt32(A); + } + if (corecursive_ != null) { + output.WriteRawTag(18); + output.WriteMessage(Corecursive); + } + if (optionalCorecursive_ != null) { + output.WriteRawTag(26); + output.WriteMessage(OptionalCorecursive); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasA) { + output.WriteRawTag(8); + output.WriteInt32(A); + } + if (corecursive_ != null) { + output.WriteRawTag(18); + output.WriteMessage(Corecursive); + } + if (optionalCorecursive_ != null) { + output.WriteRawTag(26); + output.WriteMessage(OptionalCorecursive); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasA) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(A); + } + if (corecursive_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Corecursive); + } + if (optionalCorecursive_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(OptionalCorecursive); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(NestedMessage other) { + if (other == null) { + return; + } + if (other.HasA) { + A = other.A; + } + if (other.corecursive_ != null) { + if (corecursive_ == null) { + Corecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + Corecursive.MergeFrom(other.Corecursive); + } + if (other.optionalCorecursive_ != null) { + if (optionalCorecursive_ == null) { + OptionalCorecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + OptionalCorecursive.MergeFrom(other.OptionalCorecursive); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 8: { + A = input.ReadInt32(); + break; + } + case 18: { + if (corecursive_ == null) { + Corecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(Corecursive); + break; + } + case 26: { + if (optionalCorecursive_ == null) { + OptionalCorecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(OptionalCorecursive); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 8: { + A = input.ReadInt32(); + break; + } + case 18: { + if (corecursive_ == null) { + Corecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(Corecursive); + break; + } + case 26: { + if (optionalCorecursive_ == null) { + OptionalCorecursive = new global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2(); + } + input.ReadMessage(OptionalCorecursive); + break; + } + } + } + } + #endif + + } + + /// + /// groups + /// + public sealed partial class Data : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Data()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Descriptor.NestedTypes[1]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Data() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Data(Data other) : this() { + _hasBits0 = other._hasBits0; + groupInt32_ = other.groupInt32_; + groupUint32_ = other.groupUint32_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Data Clone() { + return new Data(this); + } + + /// Field number for the "group_int32" field. + public const int GroupInt32FieldNumber = 202; + private readonly static int GroupInt32DefaultValue = 0; + + private int groupInt32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int GroupInt32 { + get { if ((_hasBits0 & 1) != 0) { return groupInt32_; } else { return GroupInt32DefaultValue; } } + set { + _hasBits0 |= 1; + groupInt32_ = value; + } + } + /// Gets whether the "group_int32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasGroupInt32 { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "group_int32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearGroupInt32() { + _hasBits0 &= ~1; + } + + /// Field number for the "group_uint32" field. + public const int GroupUint32FieldNumber = 203; + private readonly static uint GroupUint32DefaultValue = 0; + + private uint groupUint32_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint GroupUint32 { + get { if ((_hasBits0 & 2) != 0) { return groupUint32_; } else { return GroupUint32DefaultValue; } } + set { + _hasBits0 |= 2; + groupUint32_ = value; + } + } + /// Gets whether the "group_uint32" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasGroupUint32 { + get { return (_hasBits0 & 2) != 0; } + } + /// Clears the value of the "group_uint32" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearGroupUint32() { + _hasBits0 &= ~2; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as Data); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(Data other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (GroupInt32 != other.GroupInt32) return false; + if (GroupUint32 != other.GroupUint32) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasGroupInt32) hash ^= GroupInt32.GetHashCode(); + if (HasGroupUint32) hash ^= GroupUint32.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasGroupInt32) { + output.WriteRawTag(208, 12); + output.WriteInt32(GroupInt32); + } + if (HasGroupUint32) { + output.WriteRawTag(216, 12); + output.WriteUInt32(GroupUint32); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasGroupInt32) { + output.WriteRawTag(208, 12); + output.WriteInt32(GroupInt32); + } + if (HasGroupUint32) { + output.WriteRawTag(216, 12); + output.WriteUInt32(GroupUint32); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasGroupInt32) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(GroupInt32); + } + if (HasGroupUint32) { + size += 2 + pb::CodedOutputStream.ComputeUInt32Size(GroupUint32); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(Data other) { + if (other == null) { + return; + } + if (other.HasGroupInt32) { + GroupInt32 = other.GroupInt32; + } + if (other.HasGroupUint32) { + GroupUint32 = other.GroupUint32; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + case 1612: + return; + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 1616: { + GroupInt32 = input.ReadInt32(); + break; + } + case 1624: { + GroupUint32 = input.ReadUInt32(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + case 1612: + return; + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 1616: { + GroupInt32 = input.ReadInt32(); + break; + } + case 1624: { + GroupUint32 = input.ReadUInt32(); + break; + } + } + } + } + #endif + + } + + /// + /// message_set test case. + /// + public sealed partial class MessageSetCorrect : pb::IExtendableMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new MessageSetCorrect()); + private pb::UnknownFieldSet _unknownFields; + private pb::ExtensionSet _extensions; + private pb::ExtensionSet _Extensions { get { return _extensions; } } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Descriptor.NestedTypes[2]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrect() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrect(MessageSetCorrect other) : this() { + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + _extensions = pb::ExtensionSet.Clone(other._extensions); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrect Clone() { + return new MessageSetCorrect(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as MessageSetCorrect); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(MessageSetCorrect other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (!Equals(_extensions, other._extensions)) { + return false; + } + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (_extensions != null) { + hash ^= _extensions.GetHashCode(); + } + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (_extensions != null) { + _extensions.WriteTo(output); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (_extensions != null) { + _extensions.WriteTo(ref output); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (_extensions != null) { + size += _extensions.CalculateSize(); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(MessageSetCorrect other) { + if (other == null) { + return; + } + pb::ExtensionSet.MergeFrom(ref _extensions, other._extensions); + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + if (!pb::ExtensionSet.TryMergeFieldFrom(ref _extensions, input)) { + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + } + break; + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + if (!pb::ExtensionSet.TryMergeFieldFrom(ref _extensions, ref input)) { + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + } + break; + } + } + } + #endif + + public TValue GetExtension(pb::Extension extension) { + return pb::ExtensionSet.Get(ref _extensions, extension); + } + public pbc::RepeatedField GetExtension(pb::RepeatedExtension extension) { + return pb::ExtensionSet.Get(ref _extensions, extension); + } + public pbc::RepeatedField GetOrInitializeExtension(pb::RepeatedExtension extension) { + return pb::ExtensionSet.GetOrInitialize(ref _extensions, extension); + } + public void SetExtension(pb::Extension extension, TValue value) { + pb::ExtensionSet.Set(ref _extensions, extension, value); + } + public bool HasExtension(pb::Extension extension) { + return pb::ExtensionSet.Has(ref _extensions, extension); + } + public void ClearExtension(pb::Extension extension) { + pb::ExtensionSet.Clear(ref _extensions, extension); + } + public void ClearExtension(pb::RepeatedExtension extension) { + pb::ExtensionSet.Clear(ref _extensions, extension); + } + + } + + public sealed partial class MessageSetCorrectExtension1 : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new MessageSetCorrectExtension1()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Descriptor.NestedTypes[3]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension1() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension1(MessageSetCorrectExtension1 other) : this() { + str_ = other.str_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension1 Clone() { + return new MessageSetCorrectExtension1(this); + } + + /// Field number for the "str" field. + public const int StrFieldNumber = 25; + private readonly static string StrDefaultValue = ""; + + private string str_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Str { + get { return str_ ?? StrDefaultValue; } + set { + str_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "str" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasStr { + get { return str_ != null; } + } + /// Clears the value of the "str" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearStr() { + str_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as MessageSetCorrectExtension1); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(MessageSetCorrectExtension1 other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Str != other.Str) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasStr) hash ^= Str.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasStr) { + output.WriteRawTag(202, 1); + output.WriteString(Str); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasStr) { + output.WriteRawTag(202, 1); + output.WriteString(Str); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasStr) { + size += 2 + pb::CodedOutputStream.ComputeStringSize(Str); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(MessageSetCorrectExtension1 other) { + if (other == null) { + return; + } + if (other.HasStr) { + Str = other.Str; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 202: { + Str = input.ReadString(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 202: { + Str = input.ReadString(); + break; + } + } + } + } + #endif + + #region Extensions + /// Container for extensions for other messages declared in the MessageSetCorrectExtension1 message type. + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static partial class Extensions { + public static readonly pb::Extension MessageSetExtension = + new pb::Extension(1547769, pb::FieldCodec.ForMessage(12382154, global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension1.Parser)); + } + #endregion + + } + + public sealed partial class MessageSetCorrectExtension2 : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new MessageSetCorrectExtension2()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Descriptor.NestedTypes[4]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension2() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension2(MessageSetCorrectExtension2 other) : this() { + _hasBits0 = other._hasBits0; + i_ = other.i_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public MessageSetCorrectExtension2 Clone() { + return new MessageSetCorrectExtension2(this); + } + + /// Field number for the "i" field. + public const int IFieldNumber = 9; + private readonly static int IDefaultValue = 0; + + private int i_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int I { + get { if ((_hasBits0 & 1) != 0) { return i_; } else { return IDefaultValue; } } + set { + _hasBits0 |= 1; + i_ = value; + } + } + /// Gets whether the "i" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasI { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "i" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearI() { + _hasBits0 &= ~1; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as MessageSetCorrectExtension2); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(MessageSetCorrectExtension2 other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (I != other.I) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasI) hash ^= I.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasI) { + output.WriteRawTag(72); + output.WriteInt32(I); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasI) { + output.WriteRawTag(72); + output.WriteInt32(I); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasI) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(I); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(MessageSetCorrectExtension2 other) { + if (other == null) { + return; + } + if (other.HasI) { + I = other.I; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 72: { + I = input.ReadInt32(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 72: { + I = input.ReadInt32(); + break; + } + } + } + } + #endif + + #region Extensions + /// Container for extensions for other messages declared in the MessageSetCorrectExtension2 message type. + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static partial class Extensions { + public static readonly pb::Extension MessageSetExtension = + new pb::Extension(4135312, pb::FieldCodec.ForMessage(33082498, global::ProtobufTestMessages.Proto2.TestAllRequiredTypesProto2.Types.MessageSetCorrectExtension2.Parser)); + } + #endregion + + } + + } + #endregion + + } + #endregion } diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 2dc8612f630d03468aece7c8745683dc576f1aa6..c783813c973d8f403a068aa7e9eeba61af5fec28 100644 GIT binary patch delta 6667 zcmZ{pYiv}<7020g=kBiWdLN!$+jn>USg*Y{*dZY{ID`4$nE&8)xb-fP+0>}`^?hRB#$6Hp`(lCZu&R9Qh~ictHeU4gHc02cXlcZbBd$^}Z?$l8$EsBVJ%c!K zDCyg!k&)g_C7UBY()mpJ_Yt!-T$Bf`+;fLzm#8T(xien$Ch)f{TMhJV92ze5ZyNMr z$sCnGyTcyuRuSvG05$hrE2T!%I!c3MZ`;*aM67e|eT!(n%Q6eS^XDz$X~gnAtJ-HM zeHT&#@ptaBrSiy1X?ITdoH<@Sdp7r%-YVI)sOoA8Q!%NaUS(ev!`KMDTW<{w_t|L~ zAJ99t27bO#oWQrnhDZ9}EhYSNWZl|6eqUPe{f9Laer08bHjnlX4fYQ7_%r>9b-4gG z#hbXI_~5IdXsgLZrK(E2!D4;AqrLXw0K~1%759l3qh?2d%D;}9kK)hgQFA%|Oh(Nv z{Ar4rPvTEc%(eayGgrVm6*HIMPdIL_4B~-WW-Tm#8aM6o#<-b*e@onr+8s9+!kUVk z58+QX;fg$yFdumtr*T>EmqR1J80#%o_LVmFjtz{uhcYuBu%h=Ktx7A*DnIwFr)r`P>QK_Ob9}%`se<&(ony?Y|867rlC*)%*f zwpqM(@uE6{m6bnBn|Wk-Ds6VL?PO)k|IC=#6i-+N&XspOisGN0=gN;}&2-@T`|AHr z)@-Qw;k!57g}wUVV14;Q)~qN`WXxLjvE*mUIrzeES^7KA|17vI@N>Z4@cLkQJe2tK zrSepRc`@>rzXhxiHw*KAXYBk}qxs1Sr_v%8;=#`eu7DM>3>FG-_Bn;~R?K27WX$8i zFNpq5h849;VPp6UvT89L6ei8%>`P*4D{h56o4Jlk?J$KS%C!TCFDaRV!;y0g;i6c(*z!&V%|k6QEQE+!rRiiQ? zzl-wC;0kKM;cPdsa#Uh2`!cYLz6h!7vqxK-xo*Dm4ny z9^y?_)DZ`pL{auoOF^y**7q7^00mQ_44}T(C{efXU6e~Q+_^ppQ}=2T^GrozAUS(A ziUGA(qZm+o^`a2GPoqqrU@Da9j{5nTAnns66Qq5bgpSV5zdYf;d0@YF#-}y%W|X*)De9W zB0wF{`xgP~2(`3IR3&FeiPVl%0tHhosuHB56p2frN|26HqF$LG9i?1elA_#U$MmYA zK*3a~C{V}rqM|?@(~F7%bxbcR#@TUQsTfEw6)6VNab2kxNXNZOIpXn)0d-uLD$dyn zqHek5I8ZPZDvr~8f+DsW1?mK)7bz2@6V$at=9l2?B+&{vm;ec;A|*gN=}<(yni4>r zq});!4b(|`;z^0B;_MWW(y0Oprb1N#b&4X@a%vSwr>Le~nIN5_g^x%Q=j=3*lyUId za&3j;K%J&Yot(>oI!*PR$^_~(EqP3$lAN6(S}O;WAi-3mB#M4UZ(kCmGbFCJ?g>nS zbcWiUhm?)y?+h(mN#qCOa(*f94%We zNj03ECt4>5Ye0gjNHrjxr--9aH9(!G)|Dz6sPnYuX)7(RMab_0k)5f`mEIdI4UXUr zKVo?*CK}@fiq?AbOwr_F%I;^14u64C;>IM}T`dn@B>K0yGf*u<|AR*@sun^Q^>b1S z*hQ*R$Ep@&R$45|#<)+&e3l3bU-KwhF`gS-vFfyG>+rWS7-!9c)eVr_na=;d%|rs(FEDVo#~nBrWf zR8D&Uy-dxmR)bqEo3krK%QK0b{1K6Rw|8Kyr1B&i5(rS$vmtOrSI-9i3RN}89ks!} zA{Oi&LmS8N3Kg6ul#Nz)g%&N7XD7|sReg5S5Q3?M(h$0;h0+kZs)f=Jx~hfJ5V}f* zr=?CBI#+4YTB+k?IJ>4_Q5guqR6-dDUDFR%213^ezfrstG7!2ZswGVbU86Sj_&^UFjlNr@N&fiGpdIWOeItgp&MGL9tqvh zI`z=G;p%wzzj_GW&>7W3=!VXyfwKuB^=4>*5KJZ10HFze$!UPl1SRvbHVqJ(pn`YT z3Q+@eCTPJ^YBYrYJ3)(A$kH`(c9Uqm9BqUUOeNF^p_^K$5kfb$P$PtHYN195-J}K2 zDHIaANsFJAIyufJiB^k2JpYt~5NsusgV3bDA>|-6Nh$T*(tWqX~ zCaHU+6l&sZis%J7*aRJzN~Z}rQ|Q0yQr85bDJ|3lp(!oY1fePFcwVs}G)3L(rBI%; zTSQCbU>-s+l~5i+xAfzhht4haxZ?hshtMtkyyhWviwdgMAvCQIT{DEH^`UEq&a^(<&Cr=9r%g4lW(ZAFXSWn;;cP}X+!hGIR6;Efn$efM z76{EyLfsi!AT&db>I}9(Xoi}*WN})cGeZx{cI!}qv)j7g79a#u2^Ao8TW3^&&~2Sj z0YbNRMg<7nrnco$r~skc)cJ%=sFkx>qGfWh6*@4j*OU#RS-nTC5Smqc6rxrL&FU>` zh0rV&9#?ZAG)s$?T8l)l#_j(Ol~2EFKGYqmK&QJy5!sw8L=%H4@5U9PIp3jVtu%sH?T{R2M7cq#@#<-f$*Wy2%!bH=sZx33|7VcsRCK4Xx$7u=9(WAI19e;9ZlTc9s~S^h|tXw%~3ixK3(4-EDj!;~v9f{@r^ z1fQ2g*dTsj;O^*s7=Z&-Mnp_Ia242M#M`w4B3q1(mD+()Z!y+5>y?VQEo?Pdy^3%z z3vj5$Zn&3)t%l#&;Q(~2fm@GUhkIGrYGmrb6qRN?_>n=sm8rr;{6~hLDMO~t#|Hbv zPnC&Ox9L=2Bh`;}s&F7xM5t8ZK&snxs&GJMn@$xDq`J*mqcUY8)9r@$Icud@rkg4} lDpfeJ%i9hAdlnpkZa3WTS>kgR9Dr^&YVDmpZ Date: Mon, 6 Mar 2023 10:19:02 -0800 Subject: [PATCH 165/463] Automated rollback of commit 8f3ae17b3d248456691334de7a3901003acae387. PiperOrigin-RevId: 514449010 --- src/google/protobuf/descriptor.cc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index efcd86e35895..9827ac1376a1 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8417,18 +8417,7 @@ bool HasPreservingUnknownEnumSemantics(const FieldDescriptor* field) { } bool HasHasbit(const FieldDescriptor* field) { - // This predicate includes proto3 message fields only if they have "optional". - // Foo submsg1 = 1; // HasHasbit() == false - // optional Foo submsg2 = 2; // HasHasbit() == true - // This is slightly odd, as adding "optional" to a singular proto3 field does - // not change the semantics or API. However whenever any field in a message - // has a hasbit, it forces reflection to include hasbit offsets for *all* - // fields, even if almost all of them are set to -1 (no hasbit). So to avoid - // causing a sudden size regression for ~all proto3 messages, we give proto3 - // message fields a hasbit only if "optional" is present. If the user is - // explicitly writing "optional", it is likely they are writing it on - // primitive fields also. - return (field->has_optional_keyword() || field->is_required()) && + return field->has_presence() && !field->real_containing_oneof() && !field->options().weak(); } From 804ec94a1634d04019dbf368ea6ffbfc24bbb2a5 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 6 Mar 2023 10:20:14 -0800 Subject: [PATCH 166/463] Add release tests for statically linked binaries For now, this only covers linux on the two architectures we have testing support for. However, it serves as a good sanity check and can be expanded in the future. PiperOrigin-RevId: 514449399 --- .github/workflows/test_cpp.yml | 45 ++++++++++++++++++++++++++++--- .github/workflows/test_java.yml | 2 +- .github/workflows/test_python.yml | 2 +- BUILD.bazel | 1 + CMakeLists.txt | 11 +++++--- build_defs/BUILD.bazel | 1 - cmake/protobuf-generate.cmake | 4 +-- 7 files changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index b107eafea327..82d71fb0b1be 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -32,7 +32,7 @@ jobs: targets: "//src/..." - config: { name: "aarch64" } targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733" - config: { name: "Bazel4" } targets: "//src/..." image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-6361b3a6e5c97e9951d03a4de28542fc45f1adab" @@ -51,6 +51,45 @@ jobs: bazel-cache: cpp_linux/${{ matrix.config.name }} bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} --distinct_host_configuration=false + linux-release: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + arch: [x86_64, aarch64] + name: Linux Release ${{ matrix.arch}} + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + - name: Cross compile protoc for ${{ matrix.arch }} + id: cross-compile + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + architecture: linux-${{ matrix.arch }} + - name: Setup ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 + with: + cache-prefix: linux-release-${{ matrix.arch }} + - name: Run tests + uses: protocolbuffers/protobuf-ci/docker@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-3af05275178e16af30961976af126eabbbb2c733 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + skip-staleness-check: true + entrypoint: bash + command: > + -c "set -ex; + ccache -z; + cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_WITH_ZLIB=OFF ${{ env.CCACHE_CMAKE_FLAGS }}; + cmake --build . --parallel 20; + ctest --parallel 20; + ccache -s -v" + linux-cmake: strategy: fail-fast: false # Don't cancel all jobs if one fails. @@ -91,7 +130,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} ${{ env.CCACHE_CMAKE_FLAGS }} @@ -113,7 +152,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} \&\& diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index da66c4623759..67e2e3e006eb 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -24,7 +24,7 @@ jobs: targets: //java/... //java/internal:java_version - name: aarch64 version: 'aarch64' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733 targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test name: Linux ${{ matrix.name }} diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index a94018c99aa2..36f4e5ab9648 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -27,7 +27,7 @@ jobs: targets: //python/... //python:aarch64_test # TODO(b/262628111) Enable this once conformance tests are fixed. flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733 name: Linux ${{ matrix.type }} ${{ matrix.version }} runs-on: ubuntu-latest diff --git a/BUILD.bazel b/BUILD.bazel index b0b29f812883..3f8e5635e898 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -179,6 +179,7 @@ cc_binary( linkopts = LINK_OPTS, features = select({ # This isn't possible on mac because there is no static library for lcrt0.o + "@platforms//os:osx": [], "//build_defs:config_osx": [], # When cross-compiling we need to statically link all C++ libraries. "//conditions:default": ["fully_static_link"], diff --git a/CMakeLists.txt b/CMakeLists.txt index cc574f90a7a4..93484911d95e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,9 +39,6 @@ endif() # Options option(protobuf_INSTALL "Install protobuf binaries and files" ON) -if(WITH_PROTOC) - set(protobuf_PROTOC_EXE ${WITH_PROTOC} CACHE FILEPATH "Protocol Buffer Compiler executable" FORCE) -endif() option(protobuf_BUILD_TESTS "Build tests" ON) option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF) option(protobuf_BUILD_EXAMPLES "Build examples" OFF) @@ -65,6 +62,14 @@ option(protobuf_WITH_ZLIB "Build with zlib support" ${protobuf_WITH_ZLIB_DEFAULT set(protobuf_DEBUG_POSTFIX "d" CACHE STRING "Default debug postfix") mark_as_advanced(protobuf_DEBUG_POSTFIX) + +if(WITH_PROTOC) + set(protobuf_PROTOC_EXE protoc) + set(protobuf_BUILD_PROTOC_BINARIES OFF) + add_executable(protoc IMPORTED) + set_property(TARGET protoc PROPERTY IMPORTED_LOCATION ${WITH_PROTOC}) +endif() + # User options include(${protobuf_SOURCE_DIR}/cmake/protobuf-options.cmake) diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel index 0926b856d83e..855893f33e40 100644 --- a/build_defs/BUILD.bazel +++ b/build_defs/BUILD.bazel @@ -97,7 +97,6 @@ config_setting( selects.config_setting_group( name = "config_osx", match_any = [ - "@platforms//os:osx", ":config_osx_aarch64", ":config_osx_x86_64", ] diff --git a/cmake/protobuf-generate.cmake b/cmake/protobuf-generate.cmake index 10fd616a5341..5a4ff391e0ed 100644 --- a/cmake/protobuf-generate.cmake +++ b/cmake/protobuf-generate.cmake @@ -137,9 +137,9 @@ function(protobuf_generate) add_custom_command( OUTPUT ${_generated_srcs} - COMMAND protobuf::protoc + COMMAND ${protobuf_PROTOC_EXE} ARGS ${protobuf_generate_PROTOC_OPTIONS} --${protobuf_generate_LANGUAGE}_out ${_plugin_options}:${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file} - DEPENDS ${_abs_file} protobuf::protoc ${protobuf_generate_DEPENDENCIES} + DEPENDS ${_abs_file} ${protobuf_PROTOC_EXE} ${protobuf_generate_DEPENDENCIES} COMMENT ${_comment} VERBATIM ) endforeach() From 8c3fac39131d97fd9f648155de5498af1b2bf316 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 Mar 2023 18:37:28 +0000 Subject: [PATCH 167/463] Auto-generate files after cl/514449399 --- src/google/protobuf/api.pb.cc | 67 ++++++++---- src/google/protobuf/api.pb.h | 29 +++--- src/google/protobuf/type.pb.cc | 185 +++++++++++++++++++++------------ src/google/protobuf/type.pb.h | 87 ++++++++-------- 4 files changed, 224 insertions(+), 144 deletions(-) diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index 649d48dc6054..c7f399399f76 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -21,7 +21,9 @@ namespace _pbi = ::PROTOBUF_NAMESPACE_ID::internal; PROTOBUF_NAMESPACE_OPEN PROTOBUF_CONSTEXPR Api::Api( ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_.methods_)*/{} + /*decltype(_impl_._has_bits_)*/{} + , /*decltype(_impl_._cached_size_)*/{} + , /*decltype(_impl_.methods_)*/{} , /*decltype(_impl_.options_)*/{} , /*decltype(_impl_.mixins_)*/{} , /*decltype(_impl_.name_)*/ { @@ -34,8 +36,7 @@ PROTOBUF_CONSTEXPR Api::Api( , /*decltype(_impl_.source_context_)*/nullptr , /*decltype(_impl_.syntax_)*/ 0 - - , /*decltype(_impl_._cached_size_)*/{}} {} +} {} struct ApiDefaultTypeInternal { PROTOBUF_CONSTEXPR ApiDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ApiDefaultTypeInternal() {} @@ -107,7 +108,7 @@ static constexpr const ::_pb::ServiceDescriptor** file_level_service_descriptors_google_2fprotobuf_2fapi_2eproto = nullptr; const ::uint32_t TableStruct_google_2fprotobuf_2fapi_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE( protodesc_cold) = { - ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Api, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Api, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -122,6 +123,13 @@ const ::uint32_t TableStruct_google_2fprotobuf_2fapi_2eproto::offsets[] PROTOBUF PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Api, _impl_.source_context_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Api, _impl_.mixins_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Api, _impl_.syntax_), + ~0u, + ~0u, + ~0u, + ~0u, + 0, + ~0u, + ~0u, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Method, _internal_metadata_), ~0u, // no _extensions_ @@ -151,9 +159,9 @@ const ::uint32_t TableStruct_google_2fprotobuf_2fapi_2eproto::offsets[] PROTOBUF static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Api)}, - { 15, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Method)}, - { 30, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Mixin)}, + { 0, 15, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Api)}, + { 22, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Method)}, + { 37, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Mixin)}, }; static const ::_pb::Message* const file_default_instances[] = { @@ -228,7 +236,13 @@ PROTOBUF_NAMESPACE_OPEN class Api::_Internal { public: + using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr ::int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(Api, _impl_._has_bits_); static const ::PROTOBUF_NAMESPACE_ID::SourceContext& source_context(const Api* msg); + static void set_has_source_context(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } }; const ::PROTOBUF_NAMESPACE_ID::SourceContext& @@ -239,10 +253,8 @@ void Api::clear_options() { _impl_.options_.Clear(); } void Api::clear_source_context() { - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; - } - _impl_.source_context_ = nullptr; + if (_impl_.source_context_ != nullptr) _impl_.source_context_->Clear(); + _impl_._has_bits_[0] &= ~0x00000001u; } Api::Api(::PROTOBUF_NAMESPACE_ID::Arena* arena) : ::PROTOBUF_NAMESPACE_ID::Message(arena) { @@ -253,7 +265,9 @@ Api::Api(const Api& from) : ::PROTOBUF_NAMESPACE_ID::Message() { Api* const _this = this; (void)_this; new (&_impl_) Impl_{ - decltype(_impl_.methods_){from._impl_.methods_} + decltype(_impl_._has_bits_){from._impl_._has_bits_} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.methods_){from._impl_.methods_} , decltype(_impl_.options_){from._impl_.options_} , decltype(_impl_.mixins_){from._impl_.mixins_} , decltype(_impl_.name_) {} @@ -262,8 +276,7 @@ Api::Api(const Api& from) , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) {} - - , /*decltype(_impl_._cached_size_)*/{}}; + }; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); @@ -280,7 +293,7 @@ Api::Api(const Api& from) if (!from._internal_version().empty()) { _this->_impl_.version_.Set(from._internal_version(), _this->GetArenaForAllocation()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_impl_.source_context_ = new ::PROTOBUF_NAMESPACE_ID::SourceContext(*from._impl_.source_context_); } _this->_impl_.syntax_ = from._impl_.syntax_; @@ -290,7 +303,9 @@ Api::Api(const Api& from) inline void Api::SharedCtor(::_pb::Arena* arena) { (void)arena; new (&_impl_) Impl_{ - decltype(_impl_.methods_){arena} + decltype(_impl_._has_bits_){} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.methods_){arena} , decltype(_impl_.options_){arena} , decltype(_impl_.mixins_){arena} , decltype(_impl_.name_) {} @@ -300,7 +315,6 @@ inline void Api::SharedCtor(::_pb::Arena* arena) { , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) { 0 } - , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -346,16 +360,19 @@ void Api::Clear() { _impl_.mixins_.Clear(); _impl_.name_.ClearToEmpty(); _impl_.version_.ClearToEmpty(); - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + ABSL_DCHECK(_impl_.source_context_ != nullptr); + _impl_.source_context_->Clear(); } - _impl_.source_context_ = nullptr; _impl_.syntax_ = 0; + _impl_._has_bits_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* Api::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; while (!ctx->Done(&ptr)) { ::uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); @@ -459,6 +476,7 @@ const char* Api::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { CHK_(ptr != nullptr); } // while message_done: + _impl_._has_bits_.Or(has_bits); return ptr; failure: ptr = nullptr; @@ -504,8 +522,9 @@ ::uint8_t* Api::_InternalSerialize( target = stream->WriteStringMaybeAliased(4, _s, target); } + cached_has_bits = _impl_._has_bits_[0]; // .google.protobuf.SourceContext source_context = 5; - if (this->_internal_has_source_context()) { + if (cached_has_bits & 0x00000001u) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, _Internal::source_context(this), _Internal::source_context(this).GetCachedSize(), target, stream); @@ -576,7 +595,8 @@ ::size_t Api::ByteSizeLong() const { } // .google.protobuf.SourceContext source_context = 5; - if (this->_internal_has_source_context()) { + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.source_context_); @@ -615,7 +635,7 @@ void Api::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_N if (!from._internal_version().empty()) { _this->_internal_set_version(from._internal_version()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_internal_mutable_source_context()->::PROTOBUF_NAMESPACE_ID::SourceContext::MergeFrom( from._internal_source_context()); } @@ -641,6 +661,7 @@ void Api::InternalSwap(Api* other) { auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); + swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); _impl_.methods_.InternalSwap(&other->_impl_.methods_); _impl_.options_.InternalSwap(&other->_impl_.options_); _impl_.mixins_.InternalSwap(&other->_impl_.mixins_); diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 8a91760d40b1..2e0f0d281c84 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -308,10 +308,6 @@ class PROTOBUF_EXPORT Api final : public: // .google.protobuf.SourceContext source_context = 5; bool has_source_context() const; - private: - bool _internal_has_source_context() const; - - public: void clear_source_context() ; const ::PROTOBUF_NAMESPACE_ID::SourceContext& source_context() const; PROTOBUF_NODISCARD ::PROTOBUF_NAMESPACE_ID::SourceContext* release_source_context(); @@ -342,6 +338,8 @@ class PROTOBUF_EXPORT Api final : typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::PROTOBUF_NAMESPACE_ID::Method > methods_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::PROTOBUF_NAMESPACE_ID::Option > options_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::PROTOBUF_NAMESPACE_ID::Mixin > mixins_; @@ -349,7 +347,6 @@ class PROTOBUF_EXPORT Api final : ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr version_; ::PROTOBUF_NAMESPACE_ID::SourceContext* source_context_; int syntax_; - mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_google_2fprotobuf_2fapi_2eproto; @@ -993,11 +990,10 @@ inline void Api::set_allocated_version(std::string* value) { } // .google.protobuf.SourceContext source_context = 5; -inline bool Api::_internal_has_source_context() const { - return this != internal_default_instance() && _impl_.source_context_ != nullptr; -} inline bool Api::has_source_context() const { - return _internal_has_source_context(); + bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; + PROTOBUF_ASSUME(!value || _impl_.source_context_ != nullptr); + return value; } inline const ::PROTOBUF_NAMESPACE_ID::SourceContext& Api::_internal_source_context() const { const ::PROTOBUF_NAMESPACE_ID::SourceContext* p = _impl_.source_context_; @@ -1014,10 +1010,15 @@ inline void Api::unsafe_arena_set_allocated_source_context( delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.source_context_); } _impl_.source_context_ = source_context; + if (source_context) { + _impl_._has_bits_[0] |= 0x00000001u; + } else { + _impl_._has_bits_[0] &= ~0x00000001u; + } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Api.source_context) } inline ::PROTOBUF_NAMESPACE_ID::SourceContext* Api::release_source_context() { - + _impl_._has_bits_[0] &= ~0x00000001u; ::PROTOBUF_NAMESPACE_ID::SourceContext* temp = _impl_.source_context_; _impl_.source_context_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE @@ -1033,13 +1034,13 @@ inline ::PROTOBUF_NAMESPACE_ID::SourceContext* Api::release_source_context() { } inline ::PROTOBUF_NAMESPACE_ID::SourceContext* Api::unsafe_arena_release_source_context() { // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context) - + _impl_._has_bits_[0] &= ~0x00000001u; ::PROTOBUF_NAMESPACE_ID::SourceContext* temp = _impl_.source_context_; _impl_.source_context_ = nullptr; return temp; } inline ::PROTOBUF_NAMESPACE_ID::SourceContext* Api::_internal_mutable_source_context() { - + _impl_._has_bits_[0] |= 0x00000001u; if (_impl_.source_context_ == nullptr) { auto* p = CreateMaybeMessage<::PROTOBUF_NAMESPACE_ID::SourceContext>(GetArenaForAllocation()); _impl_.source_context_ = p; @@ -1064,9 +1065,9 @@ inline void Api::set_allocated_source_context(::PROTOBUF_NAMESPACE_ID::SourceCon source_context = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, source_context, submessage_arena); } - + _impl_._has_bits_[0] |= 0x00000001u; } else { - + _impl_._has_bits_[0] &= ~0x00000001u; } _impl_.source_context_ = source_context; // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.source_context) diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index 49c5c37c9735..2e662ff7a20e 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -21,7 +21,9 @@ namespace _pbi = ::PROTOBUF_NAMESPACE_ID::internal; PROTOBUF_NAMESPACE_OPEN PROTOBUF_CONSTEXPR Type::Type( ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_.fields_)*/{} + /*decltype(_impl_._has_bits_)*/{} + , /*decltype(_impl_._cached_size_)*/{} + , /*decltype(_impl_.fields_)*/{} , /*decltype(_impl_.oneofs_)*/{} , /*decltype(_impl_.options_)*/{} , /*decltype(_impl_.name_)*/ { @@ -30,8 +32,7 @@ PROTOBUF_CONSTEXPR Type::Type( , /*decltype(_impl_.source_context_)*/nullptr , /*decltype(_impl_.syntax_)*/ 0 - - , /*decltype(_impl_._cached_size_)*/{}} {} +} {} struct TypeDefaultTypeInternal { PROTOBUF_CONSTEXPR TypeDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~TypeDefaultTypeInternal() {} @@ -84,7 +85,9 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 FieldDefaultTypeInternal _Field_default_instance_; PROTOBUF_CONSTEXPR Enum::Enum( ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_.enumvalue_)*/{} + /*decltype(_impl_._has_bits_)*/{} + , /*decltype(_impl_._cached_size_)*/{} + , /*decltype(_impl_.enumvalue_)*/{} , /*decltype(_impl_.options_)*/{} , /*decltype(_impl_.name_)*/ { &::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized {} @@ -92,8 +95,7 @@ PROTOBUF_CONSTEXPR Enum::Enum( , /*decltype(_impl_.source_context_)*/nullptr , /*decltype(_impl_.syntax_)*/ 0 - - , /*decltype(_impl_._cached_size_)*/{}} {} +} {} struct EnumDefaultTypeInternal { PROTOBUF_CONSTEXPR EnumDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~EnumDefaultTypeInternal() {} @@ -126,12 +128,13 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 EnumValueDefaultTypeInternal _EnumValue_default_instance_; PROTOBUF_CONSTEXPR Option::Option( ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_.name_)*/ { + /*decltype(_impl_._has_bits_)*/{} + , /*decltype(_impl_._cached_size_)*/{} + , /*decltype(_impl_.name_)*/ { &::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized {} } - , /*decltype(_impl_.value_)*/nullptr - , /*decltype(_impl_._cached_size_)*/{}} {} + , /*decltype(_impl_.value_)*/nullptr} {} struct OptionDefaultTypeInternal { PROTOBUF_CONSTEXPR OptionDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~OptionDefaultTypeInternal() {} @@ -149,7 +152,7 @@ static constexpr const ::_pb::ServiceDescriptor** file_level_service_descriptors_google_2fprotobuf_2ftype_2eproto = nullptr; const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE( protodesc_cold) = { - ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Type, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Type, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -163,6 +166,12 @@ const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBU PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Type, _impl_.options_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Type, _impl_.source_context_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Type, _impl_.syntax_), + ~0u, + ~0u, + ~0u, + ~0u, + 0, + ~0u, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Field, _internal_metadata_), ~0u, // no _extensions_ @@ -181,7 +190,7 @@ const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBU PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Field, _impl_.options_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Field, _impl_.json_name_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Field, _impl_.default_value_), - ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Enum, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Enum, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -194,6 +203,11 @@ const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBU PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Enum, _impl_.options_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Enum, _impl_.source_context_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Enum, _impl_.syntax_), + ~0u, + ~0u, + ~0u, + 0, + ~0u, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::EnumValue, _internal_metadata_), ~0u, // no _extensions_ @@ -205,7 +219,7 @@ const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBU PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::EnumValue, _impl_.name_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::EnumValue, _impl_.number_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::EnumValue, _impl_.options_), - ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Option, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Option, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -215,15 +229,17 @@ const ::uint32_t TableStruct_google_2fprotobuf_2ftype_2eproto::offsets[] PROTOBU ~0u, // no sizeof(Split) PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Option, _impl_.name_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Option, _impl_.value_), + ~0u, + 0, }; static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Type)}, - { 14, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Field)}, - { 32, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Enum)}, - { 45, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValue)}, - { 56, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Option)}, + { 0, 14, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Type)}, + { 20, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Field)}, + { 38, 51, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Enum)}, + { 56, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValue)}, + { 67, 77, -1, sizeof(::PROTOBUF_NAMESPACE_ID::Option)}, }; static const ::_pb::Message* const file_default_instances[] = { @@ -420,7 +436,13 @@ bool Syntax_IsValid(int value) { class Type::_Internal { public: + using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr ::int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(Type, _impl_._has_bits_); static const ::PROTOBUF_NAMESPACE_ID::SourceContext& source_context(const Type* msg); + static void set_has_source_context(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } }; const ::PROTOBUF_NAMESPACE_ID::SourceContext& @@ -428,10 +450,8 @@ Type::_Internal::source_context(const Type* msg) { return *msg->_impl_.source_context_; } void Type::clear_source_context() { - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; - } - _impl_.source_context_ = nullptr; + if (_impl_.source_context_ != nullptr) _impl_.source_context_->Clear(); + _impl_._has_bits_[0] &= ~0x00000001u; } Type::Type(::PROTOBUF_NAMESPACE_ID::Arena* arena) : ::PROTOBUF_NAMESPACE_ID::Message(arena) { @@ -442,15 +462,16 @@ Type::Type(const Type& from) : ::PROTOBUF_NAMESPACE_ID::Message() { Type* const _this = this; (void)_this; new (&_impl_) Impl_{ - decltype(_impl_.fields_){from._impl_.fields_} + decltype(_impl_._has_bits_){from._impl_._has_bits_} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.fields_){from._impl_.fields_} , decltype(_impl_.oneofs_){from._impl_.oneofs_} , decltype(_impl_.options_){from._impl_.options_} , decltype(_impl_.name_) {} , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) {} - - , /*decltype(_impl_._cached_size_)*/{}}; + }; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); @@ -460,7 +481,7 @@ Type::Type(const Type& from) if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_impl_.source_context_ = new ::PROTOBUF_NAMESPACE_ID::SourceContext(*from._impl_.source_context_); } _this->_impl_.syntax_ = from._impl_.syntax_; @@ -470,7 +491,9 @@ Type::Type(const Type& from) inline void Type::SharedCtor(::_pb::Arena* arena) { (void)arena; new (&_impl_) Impl_{ - decltype(_impl_.fields_){arena} + decltype(_impl_._has_bits_){} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.fields_){arena} , decltype(_impl_.oneofs_){arena} , decltype(_impl_.options_){arena} , decltype(_impl_.name_) {} @@ -478,7 +501,6 @@ inline void Type::SharedCtor(::_pb::Arena* arena) { , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) { 0 } - , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -518,16 +540,19 @@ void Type::Clear() { _impl_.oneofs_.Clear(); _impl_.options_.Clear(); _impl_.name_.ClearToEmpty(); - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + ABSL_DCHECK(_impl_.source_context_ != nullptr); + _impl_.source_context_->Clear(); } - _impl_.source_context_ = nullptr; _impl_.syntax_ = 0; + _impl_._has_bits_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* Type::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; while (!ctx->Done(&ptr)) { ::uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); @@ -622,6 +647,7 @@ const char* Type::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { CHK_(ptr != nullptr); } // while message_done: + _impl_._has_bits_.Or(has_bits); return ptr; failure: ptr = nullptr; @@ -667,8 +693,9 @@ ::uint8_t* Type::_InternalSerialize( InternalWriteMessage(4, repfield, repfield.GetCachedSize(), target, stream); } + cached_has_bits = _impl_._has_bits_[0]; // .google.protobuf.SourceContext source_context = 5; - if (this->_internal_has_source_context()) { + if (cached_has_bits & 0x00000001u) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, _Internal::source_context(this), _Internal::source_context(this).GetCachedSize(), target, stream); @@ -724,7 +751,8 @@ ::size_t Type::ByteSizeLong() const { } // .google.protobuf.SourceContext source_context = 5; - if (this->_internal_has_source_context()) { + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.source_context_); @@ -760,7 +788,7 @@ void Type::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_ if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_internal_mutable_source_context()->::PROTOBUF_NAMESPACE_ID::SourceContext::MergeFrom( from._internal_source_context()); } @@ -786,6 +814,7 @@ void Type::InternalSwap(Type* other) { auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); + swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); _impl_.fields_.InternalSwap(&other->_impl_.fields_); _impl_.oneofs_.InternalSwap(&other->_impl_.oneofs_); _impl_.options_.InternalSwap(&other->_impl_.options_); @@ -1338,7 +1367,13 @@ ::PROTOBUF_NAMESPACE_ID::Metadata Field::GetMetadata() const { class Enum::_Internal { public: + using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr ::int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(Enum, _impl_._has_bits_); static const ::PROTOBUF_NAMESPACE_ID::SourceContext& source_context(const Enum* msg); + static void set_has_source_context(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } }; const ::PROTOBUF_NAMESPACE_ID::SourceContext& @@ -1346,10 +1381,8 @@ Enum::_Internal::source_context(const Enum* msg) { return *msg->_impl_.source_context_; } void Enum::clear_source_context() { - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; - } - _impl_.source_context_ = nullptr; + if (_impl_.source_context_ != nullptr) _impl_.source_context_->Clear(); + _impl_._has_bits_[0] &= ~0x00000001u; } Enum::Enum(::PROTOBUF_NAMESPACE_ID::Arena* arena) : ::PROTOBUF_NAMESPACE_ID::Message(arena) { @@ -1360,14 +1393,15 @@ Enum::Enum(const Enum& from) : ::PROTOBUF_NAMESPACE_ID::Message() { Enum* const _this = this; (void)_this; new (&_impl_) Impl_{ - decltype(_impl_.enumvalue_){from._impl_.enumvalue_} + decltype(_impl_._has_bits_){from._impl_._has_bits_} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.enumvalue_){from._impl_.enumvalue_} , decltype(_impl_.options_){from._impl_.options_} , decltype(_impl_.name_) {} , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) {} - - , /*decltype(_impl_._cached_size_)*/{}}; + }; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); @@ -1377,7 +1411,7 @@ Enum::Enum(const Enum& from) if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_impl_.source_context_ = new ::PROTOBUF_NAMESPACE_ID::SourceContext(*from._impl_.source_context_); } _this->_impl_.syntax_ = from._impl_.syntax_; @@ -1387,14 +1421,15 @@ Enum::Enum(const Enum& from) inline void Enum::SharedCtor(::_pb::Arena* arena) { (void)arena; new (&_impl_) Impl_{ - decltype(_impl_.enumvalue_){arena} + decltype(_impl_._has_bits_){} + , /*decltype(_impl_._cached_size_)*/{} + , decltype(_impl_.enumvalue_){arena} , decltype(_impl_.options_){arena} , decltype(_impl_.name_) {} , decltype(_impl_.source_context_){nullptr} , decltype(_impl_.syntax_) { 0 } - , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -1432,16 +1467,19 @@ void Enum::Clear() { _impl_.enumvalue_.Clear(); _impl_.options_.Clear(); _impl_.name_.ClearToEmpty(); - if (GetArenaForAllocation() == nullptr && _impl_.source_context_ != nullptr) { - delete _impl_.source_context_; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + ABSL_DCHECK(_impl_.source_context_ != nullptr); + _impl_.source_context_->Clear(); } - _impl_.source_context_ = nullptr; _impl_.syntax_ = 0; + _impl_._has_bits_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* Enum::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; while (!ctx->Done(&ptr)) { ::uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); @@ -1520,6 +1558,7 @@ const char* Enum::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { CHK_(ptr != nullptr); } // while message_done: + _impl_._has_bits_.Or(has_bits); return ptr; failure: ptr = nullptr; @@ -1557,8 +1596,9 @@ ::uint8_t* Enum::_InternalSerialize( InternalWriteMessage(3, repfield, repfield.GetCachedSize(), target, stream); } + cached_has_bits = _impl_._has_bits_[0]; // .google.protobuf.SourceContext source_context = 4; - if (this->_internal_has_source_context()) { + if (cached_has_bits & 0x00000001u) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(4, _Internal::source_context(this), _Internal::source_context(this).GetCachedSize(), target, stream); @@ -1608,7 +1648,8 @@ ::size_t Enum::ByteSizeLong() const { } // .google.protobuf.SourceContext source_context = 4; - if (this->_internal_has_source_context()) { + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.source_context_); @@ -1643,7 +1684,7 @@ void Enum::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_ if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } - if (from._internal_has_source_context()) { + if ((from._impl_._has_bits_[0] & 0x00000001u) != 0) { _this->_internal_mutable_source_context()->::PROTOBUF_NAMESPACE_ID::SourceContext::MergeFrom( from._internal_source_context()); } @@ -1669,6 +1710,7 @@ void Enum::InternalSwap(Enum* other) { auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); + swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); _impl_.enumvalue_.InternalSwap(&other->_impl_.enumvalue_); _impl_.options_.InternalSwap(&other->_impl_.options_); ::_pbi::ArenaStringPtr::InternalSwap(&_impl_.name_, lhs_arena, @@ -1954,7 +1996,13 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumValue::GetMetadata() const { class Option::_Internal { public: + using HasBits = decltype(std::declval
+ public enum VerificationState { + /// + /// All the extensions of the range must be declared. + /// + [pbr::OriginalName("DECLARATION")] Declaration = 0, + [pbr::OriginalName("UNVERIFIED")] Unverified = 1, + } + public sealed partial class Declaration : pb::IMessage #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE , pb::IBufferMessage @@ -2419,6 +2493,7 @@ public Declaration(Declaration other) : this() { type_ = other.type_; isRepeated_ = other.isRepeated_; reserved_ = other.reserved_; + repeated_ = other.repeated_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -2525,9 +2600,9 @@ public void ClearType() { private bool isRepeated_; /// - /// If true, indicates that the extension must be defined as repeated. - /// Otherwise the extension must be defined as optional. + /// Deprecated. Please use "repeated". /// + [global::System.ObsoleteAttribute] [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public bool IsRepeated { @@ -2538,12 +2613,14 @@ public bool IsRepeated { } } /// Gets whether the "is_repeated" field is set + [global::System.ObsoleteAttribute] [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public bool HasIsRepeated { get { return (_hasBits0 & 2) != 0; } } /// Clears the value of the "is_repeated" field + [global::System.ObsoleteAttribute] [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public void ClearIsRepeated() { @@ -2582,6 +2659,37 @@ public void ClearReserved() { _hasBits0 &= ~4; } + /// Field number for the "repeated" field. + public const int RepeatedFieldNumber = 6; + private readonly static bool RepeatedDefaultValue = false; + + private bool repeated_; + /// + /// If true, indicates that the extension must be defined as repeated. + /// Otherwise the extension must be defined as optional. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Repeated { + get { if ((_hasBits0 & 8) != 0) { return repeated_; } else { return RepeatedDefaultValue; } } + set { + _hasBits0 |= 8; + repeated_ = value; + } + } + /// Gets whether the "repeated" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRepeated { + get { return (_hasBits0 & 8) != 0; } + } + /// Clears the value of the "repeated" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRepeated() { + _hasBits0 &= ~8; + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -2602,6 +2710,7 @@ public bool Equals(Declaration other) { if (Type != other.Type) return false; if (IsRepeated != other.IsRepeated) return false; if (Reserved != other.Reserved) return false; + if (Repeated != other.Repeated) return false; return Equals(_unknownFields, other._unknownFields); } @@ -2614,6 +2723,7 @@ public override int GetHashCode() { if (HasType) hash ^= Type.GetHashCode(); if (HasIsRepeated) hash ^= IsRepeated.GetHashCode(); if (HasReserved) hash ^= Reserved.GetHashCode(); + if (HasRepeated) hash ^= Repeated.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -2652,6 +2762,10 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(40); output.WriteBool(Reserved); } + if (HasRepeated) { + output.WriteRawTag(48); + output.WriteBool(Repeated); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -2682,6 +2796,10 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(40); output.WriteBool(Reserved); } + if (HasRepeated) { + output.WriteRawTag(48); + output.WriteBool(Repeated); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -2707,6 +2825,9 @@ public int CalculateSize() { if (HasReserved) { size += 1 + 1; } + if (HasRepeated) { + size += 1 + 1; + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -2734,6 +2855,9 @@ public void MergeFrom(Declaration other) { if (other.HasReserved) { Reserved = other.Reserved; } + if (other.HasRepeated) { + Repeated = other.Repeated; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -2769,6 +2893,10 @@ public void MergeFrom(pb::CodedInputStream input) { Reserved = input.ReadBool(); break; } + case 48: { + Repeated = input.ReadBool(); + break; + } } } #endif @@ -2804,6 +2932,10 @@ public void MergeFrom(pb::CodedInputStream input) { Reserved = input.ReadBool(); break; } + case 48: { + Repeated = input.ReadBool(); + break; + } } } } From e6b4f0e2d6b21fa0ca9aeb44d7bdf6fc2df64f4a Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 24 Apr 2023 03:08:03 -0700 Subject: [PATCH 415/463] Delete the UPB kernel. It's been made obsolete by cl/524903073. PiperOrigin-RevId: 526587271 --- src/google/protobuf/compiler/rust/BUILD.bazel | 16 +- .../protobuf/compiler/rust/upb_kernel.cc | 204 ------------------ .../protobuf/compiler/rust/upb_kernel.h | 56 ----- 3 files changed, 1 insertion(+), 275 deletions(-) delete mode 100644 src/google/protobuf/compiler/rust/upb_kernel.cc delete mode 100644 src/google/protobuf/compiler/rust/upb_kernel.h diff --git a/src/google/protobuf/compiler/rust/BUILD.bazel b/src/google/protobuf/compiler/rust/BUILD.bazel index fb87e1d016c9..7c1f5b856882 100644 --- a/src/google/protobuf/compiler/rust/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/BUILD.bazel @@ -19,7 +19,6 @@ cc_library( ":context", ":message", ":naming", - ":upb_kernel", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler/cpp:names", @@ -96,17 +95,4 @@ cc_library( "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], -) - -cc_library( - name = "upb_kernel", - srcs = ["upb_kernel.cc"], - hdrs = ["upb_kernel.h"], - copts = COPTS, - include_prefix = "google/protobuf/compiler/rust", - visibility = ["//visibility:private"], - deps = [ - "//src/google/protobuf:protobuf_nowkt", - "@com_google_absl//absl/strings", - ], -) +) \ No newline at end of file diff --git a/src/google/protobuf/compiler/rust/upb_kernel.cc b/src/google/protobuf/compiler/rust/upb_kernel.cc deleted file mode 100644 index 69104b9f68b9..000000000000 --- a/src/google/protobuf/compiler/rust/upb_kernel.cc +++ /dev/null @@ -1,204 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "google/protobuf/compiler/rust/upb_kernel.h" - -#include -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "absl/strings/str_replace.h" -#include "absl/strings/string_view.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/io/printer.h" - -namespace google { -namespace protobuf { -namespace compiler { -namespace rust { - -// The prefix used by the UPB compiler to generate unique function names. -// TODO(b/275708201): Determine a principled way to generate names of UPB -// accessors. -std::string UpbMsgPrefix(const Descriptor* msg_descriptor) { - std::string upb_msg_prefix = msg_descriptor->full_name(); - absl::StrReplaceAll({{".", "_"}}, &upb_msg_prefix); - return upb_msg_prefix; -} - -std::string UpbThunkName(const FieldDescriptor* field, - const std::string& msg_prefix, - const std::string& accessor) { - return absl::StrCat(msg_prefix, accessor, field->name()); -} - -bool IsSupported(const FieldDescriptor* field) { - // Per v0 design document. - return field->is_optional() && !field->is_repeated(); -} - -absl::string_view RustTypeFromCppType(const FieldDescriptor::Type field_type) { - switch (field_type) { - case FieldDescriptor::Type::TYPE_BOOL: - return "bool"; - case FieldDescriptor::Type::TYPE_INT64: - return "i64"; - default: { - ABSL_LOG(FATAL) << "Unsupported field type: " << field_type; - } - } -} - -void GenScalarAccessors(const std::string& upb_msg_prefix, - const std::string& msg_name, - const FieldDescriptor* field, google::protobuf::io::Printer& p) { - if (!IsSupported(field)) { - return; - } - - p.Emit({{"Msg", msg_name}, - {"field_name", field->name()}, - {"data_type", RustTypeFromCppType(field->type())}, - {"has_thunk", UpbThunkName(field, upb_msg_prefix, "_has_")}, - {"getter_thunk", UpbThunkName(field, upb_msg_prefix, "_")}, - {"setter_thunk", UpbThunkName(field, upb_msg_prefix, "_set_")}, - {"clear_thunk", UpbThunkName(field, upb_msg_prefix, "_clear_")}}, - R"rs( - impl $Msg$ { - pub fn $field_name$(&self) -> Option<$data_type$> { - let field_present = unsafe { $has_thunk$(self.msg) }; - if !field_present { - return None; - } - let value = unsafe { $getter_thunk$(self.msg) }; - Some(value) - } - - pub fn $field_name$_set(&mut self, value: Option<$data_type$>) { - match value { - Some(value) => unsafe { $setter_thunk$(self.msg, value); }, - None => unsafe { $clear_thunk$(self.msg); } - } - } - } - - extern "C" { - fn $getter_thunk$(msg: ::__std::ptr::NonNull) -> $data_type$; - fn $has_thunk$(msg: ::__std::ptr::NonNull) -> bool; - fn $setter_thunk$( - msg: ::__std::ptr::NonNull, - value: $data_type$ - ); - fn $clear_thunk$(msg: ::__std::ptr::NonNull); - } - )rs"); -} - -void GenFieldAccessors(const Descriptor* msg_descriptor, - google::protobuf::io::Printer& p) { - auto upb_msg_prefix = UpbMsgPrefix(msg_descriptor); - - for (int i = 0; i < msg_descriptor->field_count(); ++i) { - auto field = msg_descriptor->field(i); - - switch (field->type()) { - case FieldDescriptor::Type::TYPE_INT64: - case FieldDescriptor::Type::TYPE_BOOL: - GenScalarAccessors(upb_msg_prefix, msg_descriptor->name(), field, p); - break; - default: - // Not implemented type. - break; - } - } -} - -void GenMessageFunctions(const Descriptor* msg_descriptor, - google::protobuf::io::Printer& p) { - p.Emit({{"Msg", msg_descriptor->name()}, - {"msg_prefix", UpbMsgPrefix(msg_descriptor)}}, - R"rs( - impl $Msg$ { - pub fn new() -> Self { - let arena = unsafe { ::__pb::Arena::new() }; - let msg = unsafe { $msg_prefix$_new(arena) }; - $Msg$ { msg, arena } - } - - pub fn serialize(&self) -> ::__pb::SerializedData { - let arena = unsafe { ::__pb::__runtime::upb_Arena_New() }; - let mut len = 0; - let chars = unsafe { $msg_prefix$_serialize(self.msg, arena, &mut len) }; - unsafe {::__pb::SerializedData::from_raw_parts(arena, chars, len)} - } - } - - extern "C" { - fn $msg_prefix$_new(arena: *mut ::__pb::Arena) -> ::__std::ptr::NonNull; - fn $msg_prefix$_serialize( - msg: ::__std::ptr::NonNull, - arena: *mut ::__pb::Arena, - len: &mut usize) -> ::__std::ptr::NonNull; - } - )rs"); -} - -void UpbKernel::Generate(const FileDescriptor* file, - google::protobuf::io::Printer& p) const { - for (int i = 0; i < file->message_type_count(); ++i) { - auto msg_descriptor = file->message_type(i); - - p.Emit({{"Msg", msg_descriptor->name()}, - { - "MsgFunctions", - [&] { GenMessageFunctions(msg_descriptor, p); }, - }, - { - "FieldAccessors", - [&] { GenFieldAccessors(msg_descriptor, p); }, - }}, - R"rs( - pub struct $Msg$ { - msg: ::__std::ptr::NonNull, - arena: *mut ::__pb::Arena, - } - - $MsgFunctions$; - $FieldAccessors$; - )rs"); - } -} - -} // namespace rust -} // namespace compiler -} // namespace protobuf -} // namespace google diff --git a/src/google/protobuf/compiler/rust/upb_kernel.h b/src/google/protobuf/compiler/rust/upb_kernel.h deleted file mode 100644 index 6fb4b45be7c9..000000000000 --- a/src/google/protobuf/compiler/rust/upb_kernel.h +++ /dev/null @@ -1,56 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_COMPILER_RUST_UPB_KERNEL_H__ -#define GOOGLE_PROTOBUF_COMPILER_RUST_UPB_KERNEL_H__ - -#include "google/protobuf/descriptor.h" -#include "google/protobuf/io/printer.h" - -namespace google { -namespace protobuf { -namespace compiler { -namespace rust { - -class UpbKernel { - public: - UpbKernel() = default; - UpbKernel(const UpbKernel&) = delete; - UpbKernel& operator=(const UpbKernel&) = delete; - - void Generate(const FileDescriptor* file, google::protobuf::io::Printer& p) const; -}; - -} // namespace rust -} // namespace compiler -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_COMPILER_RUST_UPB_KERNEL_H__ From 37c8f8195ba036aacbf22276cca3ddb095735b43 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 24 Apr 2023 10:20:09 +0000 Subject: [PATCH 416/463] Auto-generate files after cl/526587271 --- src/file_lists.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index b35561eb0ba6..da030df3d843 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -378,7 +378,6 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/naming.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/upb_kernel.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/subprocess.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/zip_writer.cc ) @@ -486,7 +485,6 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/message.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/naming.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/upb_kernel.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/scc.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/subprocess.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/zip_writer.h From 0ecb72990657f5e5ce79c8673348f2d3a656df0b Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 24 Apr 2023 03:32:46 -0700 Subject: [PATCH 417/463] Implement 'bytes' accessors for UPB :) Move the 'accessors_test.rs' to shared/ as both kernels now support the same accessor types. PiperOrigin-RevId: 526591987 --- rust/test/cpp/BUILD | 12 ---- rust/test/shared/BUILD | 20 +++++++ rust/test/{cpp => shared}/accessors_test.rs | 0 rust/test/upb/BUILD | 12 ---- rust/test/upb/accessors_test.rs | 57 ------------------- .../compiler/rust/accessors/accessors.cc | 2 +- 6 files changed, 21 insertions(+), 82 deletions(-) rename rust/test/{cpp => shared}/accessors_test.rs (100%) delete mode 100644 rust/test/upb/accessors_test.rs diff --git a/rust/test/cpp/BUILD b/rust/test/cpp/BUILD index 49d1de9b9ce5..91782bf48b2f 100644 --- a/rust/test/cpp/BUILD +++ b/rust/test/cpp/BUILD @@ -9,15 +9,3 @@ # To do that use: # * `rust_cc_proto_library` instead of `rust_proto_library`. # * `//rust:protobuf_cpp` instead of `//rust:protobuf``. - -load("@rules_rust//rust:defs.bzl", "rust_test") - -rust_test( - name = "accessors_test", - srcs = ["accessors_test.rs"], - tags = [ - # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. - "not_build:arm", - ], - deps = ["//rust/test:unittest_cc_rust_proto"], -) diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index 4740cc3afe34..c163263a3c19 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -121,3 +121,23 @@ rust_test( ], deps = ["//rust/test:unittest_upb_rust_proto"], ) + +rust_test( + name = "accessors_cpp_test", + srcs = ["accessors_test.rs"], + tags = [ + # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. + "not_build:arm", + ], + deps = ["//rust/test:unittest_cc_rust_proto"], +) + +rust_test( + name = "accessors_upb_test", + srcs = ["accessors_test.rs"], + tags = [ + # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. + "not_build:arm", + ], + deps = ["//rust/test:unittest_upb_rust_proto"], +) diff --git a/rust/test/cpp/accessors_test.rs b/rust/test/shared/accessors_test.rs similarity index 100% rename from rust/test/cpp/accessors_test.rs rename to rust/test/shared/accessors_test.rs diff --git a/rust/test/upb/BUILD b/rust/test/upb/BUILD index 8c1459cbc3a9..33f296217a5b 100644 --- a/rust/test/upb/BUILD +++ b/rust/test/upb/BUILD @@ -9,15 +9,3 @@ # To do that use: # * `rust_upb_proto_library` instead of `rust_proto_library`. # * `//rust:protobuf_upb` instead of `//rust:protobuf``. - -load("@rules_rust//rust:defs.bzl", "rust_test") - -rust_test( - name = "accessors_test", - srcs = ["accessors_test.rs"], - tags = [ - # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. - "not_build:arm", - ], - deps = ["//rust/test:unittest_upb_rust_proto"], -) diff --git a/rust/test/upb/accessors_test.rs b/rust/test/upb/accessors_test.rs deleted file mode 100644 index 670b4e8a5738..000000000000 --- a/rust/test/upb/accessors_test.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use unittest_proto::proto2_unittest::TestAllTypes; - -#[test] -fn test_optional_bool() { - let mut test_all_types: TestAllTypes = TestAllTypes::new(); - test_all_types.optional_bool_set(Some(true)); - assert_eq!(test_all_types.optional_bool(), Some(true)); - - test_all_types.optional_bool_set(Some(false)); - assert_eq!(test_all_types.optional_bool(), Some(false)); - - test_all_types.optional_bool_set(None); - assert_eq!(test_all_types.optional_bool(), None); -} - -#[test] -fn test_optional_int64() { - let mut test_all_types: TestAllTypes = TestAllTypes::new(); - test_all_types.optional_int64_set(Some(10)); - assert_eq!(test_all_types.optional_int64(), Some(10)); - - test_all_types.optional_int64_set(Some(-10)); - assert_eq!(test_all_types.optional_int64(), Some(-10)); - - test_all_types.optional_int64_set(None); - assert_eq!(test_all_types.optional_int64(), None); -} diff --git a/src/google/protobuf/compiler/rust/accessors/accessors.cc b/src/google/protobuf/compiler/rust/accessors/accessors.cc index 26ff03e2ac2b..e2e433198b21 100644 --- a/src/google/protobuf/compiler/rust/accessors/accessors.cc +++ b/src/google/protobuf/compiler/rust/accessors/accessors.cc @@ -54,7 +54,7 @@ std::unique_ptr AccessorGenerator::For( if (field.desc().is_repeated()) return nullptr; return ForSingularScalar(field); case FieldDescriptor::TYPE_BYTES: - if (field.desc().is_repeated() || field.is_upb()) return nullptr; + if (field.desc().is_repeated()) return nullptr; return ForSingularBytes(field); default: From f79e35c821a50c6a37ffe365511b892f5409ac44 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 24 Apr 2023 10:41:12 -0700 Subject: [PATCH 418/463] Fix declared dependencies for pkg-config (#12518) Closes #12439 Closes #12518 PiperOrigin-RevId: 526689125 --- cmake/install.cmake | 9 +++++++++ cmake/protobuf-lite.pc.cmake | 1 + cmake/protobuf.pc.cmake | 1 + .../utf8_range/.github/workflows/bazel_tests.yml | 9 ++++++++- third_party/utf8_range/CMakeLists.txt | 5 +++++ third_party/utf8_range/cmake/utf8_range.pc.cmake | 12 ++++++++++++ update_subtrees.sh | 2 +- 7 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 third_party/utf8_range/cmake/utf8_range.pc.cmake mode change 100644 => 100755 update_subtrees.sh diff --git a/cmake/install.cmake b/cmake/install.cmake index f7a0d9e17740..af891ca0c3d1 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -1,5 +1,14 @@ include(GNUInstallDirs) +foreach(_target IN LISTS protobuf_ABSL_USED_TARGETS) + string(REPLACE :: _ _modified_target ${_target}) + list(APPEND _pc_targets ${_modified_target}) +endforeach() +list(APPEND _pc_targets "utf8_range") + +string(JOIN " " _pc_target_list ${_pc_targets}) +set(_protobuf_PC_REQUIRES ${_pc_target_list}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf.pc.cmake ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc @ONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf-lite.pc.cmake diff --git a/cmake/protobuf-lite.pc.cmake b/cmake/protobuf-lite.pc.cmake index 9745cb89c7a7..7f742c0258dc 100644 --- a/cmake/protobuf-lite.pc.cmake +++ b/cmake/protobuf-lite.pc.cmake @@ -6,6 +6,7 @@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: Protocol Buffers Description: Google's Data Interchange Format Version: @protobuf_VERSION@ +Requires: @_protobuf_PC_REQUIRES@ Libs: -L${libdir} -lprotobuf-lite @CMAKE_THREAD_LIBS_INIT@ Cflags: -I${includedir} Conflicts: protobuf diff --git a/cmake/protobuf.pc.cmake b/cmake/protobuf.pc.cmake index f068e6926e72..21bbf47fb1d1 100644 --- a/cmake/protobuf.pc.cmake +++ b/cmake/protobuf.pc.cmake @@ -6,6 +6,7 @@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: Protocol Buffers Description: Google's Data Interchange Format Version: @protobuf_VERSION@ +Requires: @_protobuf_PC_REQUIRES@ Libs: -L${libdir} -lprotobuf @CMAKE_THREAD_LIBS_INIT@ Cflags: -I${includedir} Conflicts: protobuf-lite diff --git a/third_party/utf8_range/.github/workflows/bazel_tests.yml b/third_party/utf8_range/.github/workflows/bazel_tests.yml index ae3e970419cb..adf711859aa5 100644 --- a/third_party/utf8_range/.github/workflows/bazel_tests.yml +++ b/third_party/utf8_range/.github/workflows/bazel_tests.yml @@ -29,8 +29,15 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Setup Bazel + run: | + echo "BAZEL=bazel" >> $GITHUB_ENV + echo "USE_BAZEL_VERSION=5.1.1" >> $GITHUB_ENV + - name: Setup Bazel (macOS) + if: runner.os == 'macOS' + run: echo "BAZEL=bazelisk" >> $GITHUB_ENV - name: Install dependencies run: sudo apt update && sudo apt install -y ${{ matrix.install }} if: matrix.install != '' - name: Run tests - run: cd ${{ github.workspace }} && CC=${{ matrix.CC }} bazel test --test_output=errors ... ${{ matrix.flags }} + run: cd ${{ github.workspace }} && CC=${{ matrix.CC }} ${{ env.BAZEL }} test --test_output=errors ... ${{ matrix.flags }} diff --git a/third_party/utf8_range/CMakeLists.txt b/third_party/utf8_range/CMakeLists.txt index 69a4d8990775..344952d38cf5 100644 --- a/third_party/utf8_range/CMakeLists.txt +++ b/third_party/utf8_range/CMakeLists.txt @@ -73,6 +73,11 @@ if (utf8_range_ENABLE_INSTALL) install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}-config.cmake" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" ) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/utf8_range.pc.cmake + ${CMAKE_CURRENT_BINARY_DIR}/utf8_range.pc @ONLY) + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/utf8_range.pc + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") # Install public headers explicitly. install(FILES utf8_range.h utf8_validity.h diff --git a/third_party/utf8_range/cmake/utf8_range.pc.cmake b/third_party/utf8_range/cmake/utf8_range.pc.cmake new file mode 100644 index 000000000000..5801b43e82f1 --- /dev/null +++ b/third_party/utf8_range/cmake/utf8_range.pc.cmake @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: UTF8 Range +Description: Google's UTF8 Library +Version: 1.0 +Requires: absl_strings +Libs: -L${libdir} -lprotobuf-lite @CMAKE_THREAD_LIBS_INIT@ +Cflags: -I${includedir} +Conflicts: protobuf diff --git a/update_subtrees.sh b/update_subtrees.sh old mode 100644 new mode 100755 index df75ba3f94c5..fee33f584f17 --- a/update_subtrees.sh +++ b/update_subtrees.sh @@ -1,4 +1,4 @@ -#! /bin/bash -eux +#!/bin/bash -eux set -eux From 07b1491895582179a309ad68fca0bfa0411d9730 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Mon, 24 Apr 2023 14:40:10 -0700 Subject: [PATCH 419/463] Add better fallback behavior for retention stripping with invalid descriptors PiperOrigin-RevId: 526758886 --- src/google/protobuf/compiler/retention.cc | 18 +++- src/google/protobuf/retention_test.cc | 110 ++++++++++++++++++++++ 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/compiler/retention.cc b/src/google/protobuf/compiler/retention.cc index 6ac0876dd6cf..f34259eb1b3d 100644 --- a/src/google/protobuf/compiler/retention.cc +++ b/src/google/protobuf/compiler/retention.cc @@ -110,29 +110,39 @@ void ConvertToDynamicMessageAndStripOptions( // having to resort to DynamicMessage. StripMessage(m, path, stripped_paths); } else { + // To convert to a dynamic message, we need to serialize the original + // descriptor and parse it back again. This can fail if the descriptor is + // invalid, so in that case we try to handle it gracefully by stripping the + // original descriptor without using DynamicMessage. In this situation we + // will generally not be able to strip custom options, but we can at least + // strip built-in options. DynamicMessageFactory factory; std::unique_ptr dynamic_message( factory.GetPrototype(descriptor)->New()); std::string serialized; if (!m.SerializePartialToString(&serialized)) { ABSL_LOG_EVERY_N_SEC(ERROR, 1) - << "Failed to strip source-retention options"; + << "Failed to fully strip source-retention options"; + StripMessage(m, path, stripped_paths); return; } if (!dynamic_message->ParsePartialFromString(serialized)) { ABSL_LOG_EVERY_N_SEC(ERROR, 1) - << "Failed to strip source-retention options"; + << "Failed to fully strip source-retention options"; + StripMessage(m, path, stripped_paths); return; } StripMessage(*dynamic_message, path, stripped_paths); if (!dynamic_message->SerializePartialToString(&serialized)) { ABSL_LOG_EVERY_N_SEC(ERROR, 1) - << "Failed to strip source-retention options"; + << "Failed to fully strip source-retention options"; + StripMessage(m, path, stripped_paths); return; } if (!m.ParsePartialFromString(serialized)) { ABSL_LOG_EVERY_N_SEC(ERROR, 1) - << "Failed to strip source-retention options"; + << "Failed to fully strip source-retention options"; + StripMessage(m, path, stripped_paths); return; } } diff --git a/src/google/protobuf/retention_test.cc b/src/google/protobuf/retention_test.cc index 2f0075a5bbb8..192533cc8bbb 100644 --- a/src/google/protobuf/retention_test.cc +++ b/src/google/protobuf/retention_test.cc @@ -281,6 +281,116 @@ TEST(RetentionTest, InvalidDescriptor) { compiler::StripSourceRetentionOptions(*file_descriptor); } +TEST(RetentionTest, MissingRequiredField) { + // Retention stripping should work correctly for a descriptor that has + // options with missing required fields. + std::string proto_file = + absl::Substitute(R"( + syntax = "proto2"; + + package google.protobuf.internal; + + import "$0"; + + message WithRequiredField { + required int32 required_field = 1; + optional int32 optional_field = 2; + } + + // Option with missing required field + option (m).optional_field = 42; + + extend google.protobuf.FileOptions { + optional WithRequiredField m = 50000; + } + + message Extendee { + extensions 1 to max [ + declaration = {number: 1 full_name: ".my.ext" type: ".my.Type"} + ]; + })", + FileDescriptorSet::descriptor()->file()->name()); + io::ArrayInputStream input_stream(proto_file.data(), + static_cast(proto_file.size())); + io::ErrorCollector error_collector; + io::Tokenizer tokenizer(&input_stream, &error_collector); + compiler::Parser parser; + FileDescriptorProto file_descriptor_proto; + ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor_proto)); + file_descriptor_proto.set_name("retention.proto"); + + DescriptorPool pool; + FileDescriptorProto descriptor_proto_descriptor; + FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor); + ASSERT_NE(pool.BuildFile(descriptor_proto_descriptor), nullptr); + const FileDescriptor* file_descriptor = pool.BuildFile(file_descriptor_proto); + ASSERT_NE(file_descriptor, nullptr); + + FileDescriptorProto stripped_file = + compiler::StripSourceRetentionOptions(*file_descriptor); + ASSERT_EQ(stripped_file.message_type_size(), 2); + const DescriptorProto& extendee = stripped_file.message_type(1); + EXPECT_EQ(extendee.name(), "Extendee"); + ASSERT_EQ(extendee.extension_range_size(), 1); + EXPECT_EQ(extendee.extension_range(0).options().declaration_size(), 0); +} + +TEST(RetentionTest, InvalidRecursionDepth) { + // The excessive nesting in this proto file will make it impossible for us to + // use a DynamicMessage to strip custom options, but we should still fall + // back to stripping built-in options (specifically extension declarations). + std::string proto_file = + absl::Substitute(R"( + syntax = "proto2"; + + package google.protobuf.internal; + + import "$0"; + + message Recursive { + optional Recursive r = 1; + } + + option (r).r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r + .r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r + .r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r + .r.r.r.r.r.r.r.r.r.r.r.r = {}; + + extend google.protobuf.FileOptions { + optional Recursive r = 50000; + } + + message Extendee { + extensions 1 to max [ + declaration = {number: 1 full_name: ".my.ext" type: ".my.Type"} + ]; + })", + FileDescriptorSet::descriptor()->file()->name()); + io::ArrayInputStream input_stream(proto_file.data(), + static_cast(proto_file.size())); + io::ErrorCollector error_collector; + io::Tokenizer tokenizer(&input_stream, &error_collector); + compiler::Parser parser; + FileDescriptorProto file_descriptor_proto; + ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor_proto)); + file_descriptor_proto.set_name("retention.proto"); + + DescriptorPool pool; + FileDescriptorProto descriptor_proto_descriptor; + FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor); + ASSERT_NE(pool.BuildFile(descriptor_proto_descriptor), nullptr); + const FileDescriptor* file_descriptor = pool.BuildFile(file_descriptor_proto); + ASSERT_NE(file_descriptor, nullptr); + + FileDescriptorProto stripped_file = + compiler::StripSourceRetentionOptions(*file_descriptor); + ASSERT_EQ(stripped_file.message_type_size(), 2); + const DescriptorProto& extendee = stripped_file.message_type(1); + EXPECT_EQ(extendee.name(), "Extendee"); + ASSERT_EQ(extendee.extension_range_size(), 1); + EXPECT_EQ(extendee.extension_range(0).options().declaration_size(), 0); +} + } // namespace } // namespace internal } // namespace protobuf From d692223887d001983f4f96764e1da8c8b67860f7 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 24 Apr 2023 15:50:36 -0700 Subject: [PATCH 420/463] [WIP] Java BOM check with generated pom.xml files (#12502) This check aims to address issues like https://github.com/protocolbuffers/protobuf/issues/12170 "Java 3.22.1: broken pom.xml". Closes #12502 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12502 from suztomo:bom_check c69e2af597b1098c4e34b85e6940dcf03da631b8 PiperOrigin-RevId: 526776900 --- .github/workflows/test_java.yml | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index e1705a30d805..da1680c9ad7e 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -73,3 +73,52 @@ jobs: run: curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/linkage-monitor-latest-all-deps.jar" - name: Fails if there's new linkage errors compared with baseline run: java -Xmx2048m -jar linkage-monitor-latest-all-deps.jar com.google.cloud:libraries-bom + + protobuf-bom: + name: Protobuf Maven BOM + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + - name: Build protoc + id: build-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + architecture: linux-x86_64 + - name: Move protoc into place and clean up + run: | + mv ${{ steps.build-protoc.outputs.protoc }} protoc + sudo rm -rf _build + - name: Install snapshot version locally (not using generated pom.xml) + run: | + mvn -e -B -Dhttps.protocols=TLSv1.2 install -Dmaven.test.skip=true + working-directory: java + - name: Generate pom.xml files from the template + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-03a376b5d6ef66f827fc307716e3b841cc26b709 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: java_linux/11 + # protobuf-java and protobuf-java-util are the member of the BOM + bash: | + bazel build //java/core:core_mvn-pom //java/util:util_mvn-pom + cp bazel-bin/java/core/core_mvn-pom.xml . + cp bazel-bin/java/util/util_mvn-pom.xml . + - name: Copy the generated pom.xml files to the local Maven repository + shell: bash + run: | + LOCAL_MAVEN_GROUP_DIR="${HOME}/.m2/repository/com/google/protobuf" + VERSION=$(grep 'Bundle-Version:' ./java/core/target/classes/META-INF/MANIFEST.MF |awk '{print $2}' |tr -d '\r') + cp core_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java/${VERSION}/protobuf-java-${VERSION}.pom + cp util_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java-util/${VERSION}/protobuf-java-util-${VERSION}.pom + - name: Clean up + run: | + sudo rm -rf _build + - name: Validate Protobuf BOM + uses: googleapis/java-cloud-bom/tests/validate-bom@v26.13.0 + with: + bom-path: java/bom/pom.xml From 5233990ad79b4dc4a484bbb3fafaa7c8a715dfac Mon Sep 17 00:00:00 2001 From: Son Luong Ngoc Date: Mon, 24 Apr 2023 20:16:52 -0700 Subject: [PATCH 421/463] Fix protoc_nowkt linking on MacOS (#12320) Depending on how the local cc toolchain is setup on macos, the core foundation framework might not be available while linking protoc_nowkt. This results in a few _CF symbols being interpreted as dynamic load and is set to the binary's flat-namespace, causing runtime errors during susequent actions such as gen_wkt_cc_sources. ```bash > otool -dyld_info bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/external/com_google_protobuf/src/google/protobuf/compiler/protoc_nowkt | grep '_CF' __DATA_CONST __got 0x10035C038 0x8010000000000007 bind 0x0 flat-namespace _CFRelease __DATA_CONST __got 0x10035C040 0x8010000000000008 bind 0x0 flat-namespace _CFStringGetCString __DATA_CONST __got 0x10035C048 0x8010000000000009 bind 0x0 flat-namespace _CFStringGetLength __DATA_CONST __got 0x10035C050 0x801000000000000A bind 0x0 flat-namespace _CFStringGetMaximumSizeForEncoding __DATA_CONST __got 0x10035C058 0x801000000000000B bind 0x0 flat-namespace _CFTimeZoneCopyDefault __DATA_CONST __got 0x10035C060 0x801000000000000C bind 0x0 flat-namespace _CFTimeZoneGetName ``` Tell the linker to use CoreFoundation framework explicitly fixes this issue. ```bash > otool -dyld_info bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/external/com_google_protobuf/src/google/protobuf/compiler/protoc_nowkt | grep '_CF' __DATA_CONST __got 0x10035C038 0x8010000000000007 bind 0x0 CoreFoundation _CFRelease __DATA_CONST __got 0x10035C040 0x8010000000000008 bind 0x0 CoreFoundation _CFStringGetCString __DATA_CONST __got 0x10035C048 0x8010000000000009 bind 0x0 CoreFoundation _CFStringGetLength __DATA_CONST __got 0x10035C050 0x801000000000000A bind 0x0 CoreFoundation _CFStringGetMaximumSizeForEncoding __DATA_CONST __got 0x10035C058 0x801000000000000B bind 0x0 CoreFoundation _CFTimeZoneCopyDefault __DATA_CONST __got 0x10035C060 0x801000000000000C bind 0x0 CoreFoundation _CFTimeZoneGetName ``` closes #12173 Closes #12320 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12320 from sluongng:main 1416168f128b05d72692a3b451dd96c2b3c1cc49 PiperOrigin-RevId: 526830008 --- build_defs/cpp_opts.bzl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index f95c3e257902..5a32371dacf7 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -36,6 +36,11 @@ LINK_OPTS = select({ # Suppress linker warnings about files with no symbols defined. "-ignore:4221", ], + "@platforms//os:macos": [ + "-lpthread", + "-lm", + "-framework CoreFoundation", + ], "//conditions:default": [ "-lpthread", "-lm", From 9898418bd9188b22e9db7f94529df638e65b14f7 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 25 Apr 2023 11:10:28 -0700 Subject: [PATCH 422/463] Fix btree issue in map tests. Without any weak ordering, btree_map fails to compile on windows 2022. Closes #12289 PiperOrigin-RevId: 527019995 --- .github/workflows/test_cpp.yml | 11 +++++++++++ src/google/protobuf/map_test.inc | 2 ++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index f473c9fe42c2..c5b42be3a5d5 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -278,11 +278,20 @@ jobs: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_EXAMPLES=ON + vsversion: '2019' + - name: Windows CMake 2022 + os: windows-2022 + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_BUILD_SHARED_LIBS=OFF + -Dprotobuf_BUILD_EXAMPLES=ON + vsversion: '2022' - name: Windows CMake Shared os: windows-2019 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON + vsversion: '2019' - name: Windows CMake Install os: windows-2019 install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF @@ -290,6 +299,7 @@ jobs: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF + vsversion: '2019' name: ${{ matrix.name }} runs-on: ${{ matrix.os }} steps: @@ -305,6 +315,7 @@ jobs: uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: ${{ matrix.name }} + vsversion: ${{ matrix.vsversion }} # Install phase. - name: Configure CMake for install diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index c27c1cace5c9..209802472a1c 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -132,6 +132,8 @@ struct ConstructorTag { ConstructorTag(ConstructorTag&&) : invoked_constructor(ConstructorType::kMove) {} + bool operator<(const ConstructorTag&) const { return false; } + ConstructorType invoked_constructor; }; From 182d564a3fd5db95b36188a3c195b177e23c1c9f Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 25 Apr 2023 13:01:44 -0700 Subject: [PATCH 423/463] Move all the reflection related data of MapField into a separate structure. This saves 24 bytes on each `map` field when reflection is not used. It also delays constructing a Mutex until reflection is needed. This means we do not have to register the mutex for destruction in the arena for normal operation. PiperOrigin-RevId: 527051764 --- .../cpp/field_generators/map_field.cc | 24 +- src/google/protobuf/dynamic_message.cc | 10 - src/google/protobuf/map_field.cc | 243 ++++++++---------- src/google/protobuf/map_field.h | 210 ++++++++------- src/google/protobuf/map_field_inl.h | 25 +- src/google/protobuf/map_field_test.cc | 155 +++++------ 6 files changed, 296 insertions(+), 371 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc index dde6cff0703d..97bdb68b0c5b 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc @@ -104,8 +104,6 @@ class MapFieldGenerator : public FieldGeneratorBase { void GenerateCopyAggregateInitializer(io::Printer* printer) const override; void GenerateAggregateInitializer(io::Printer* printer) const override; void GenerateDestructorCode(io::Printer* printer) const override; - void GenerateArenaDestructorCode(io::Printer* printer) const override; - ArenaDtorNeeds NeedsArenaDestructor() const override; private: bool has_required_fields_; @@ -304,11 +302,7 @@ void MapFieldGenerator::GenerateIsInitialized(io::Printer* printer) const { void MapFieldGenerator::GenerateConstexprAggregateInitializer( io::Printer* printer) const { Formatter format(printer, variables_); - if (HasDescriptorMethods(descriptor_->file(), options_)) { - format("/*decltype($field$)*/{::_pbi::ConstantInitialized()}"); - } else { - format("/*decltype($field$)*/{}"); - } + format("/*decltype($field$)*/{}"); } void MapFieldGenerator::GenerateCopyAggregateInitializer( @@ -341,22 +335,6 @@ void MapFieldGenerator::GenerateDestructorCode(io::Printer* printer) const { format("$field$.~MapField$lite$();\n"); } -void MapFieldGenerator::GenerateArenaDestructorCode( - io::Printer* printer) const { - if (NeedsArenaDestructor() == ArenaDtorNeeds::kNone) { - return; - } - - Formatter format(printer, variables_); - // _this is the object being destructed (we are inside a static method here). - format("_this->$field$.ArenaDestruct();\n"); -} - -ArenaDtorNeeds MapFieldGenerator::NeedsArenaDestructor() const { - return HasDescriptorMethods(descriptor_->file(), options_) - ? ArenaDtorNeeds::kRequired - : ArenaDtorNeeds::kNone; -} } // namespace std::unique_ptr MakeMapGenerator( diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 680fa0b40731..985ae703c3da 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -439,11 +439,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) { new (field_ptr) DynamicMapField( type_info_->factory->GetPrototype(field->message_type()), GetArenaForAllocation()); - if (GetOwningArena() != nullptr) { - // Needs to destroy the mutex member. - static_cast(field_ptr)->OwnMutexDestructor( - *GetOwningArena()); - } } else { new (field_ptr) DynamicMapField( type_info_->factory->GetPrototype(field->message_type())); @@ -454,11 +449,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) { DynamicMapField(type_info_->factory->GetPrototypeNoLock( field->message_type()), GetArenaForAllocation()); - if (GetOwningArena() != nullptr) { - // Needs to destroy the mutex member. - static_cast(field_ptr)->OwnMutexDestructor( - *GetOwningArena()); - } } else { new (field_ptr) DynamicMapField(type_info_->factory->GetPrototypeNoLock( diff --git a/src/google/protobuf/map_field.cc b/src/google/protobuf/map_field.cc index 59c17624cb16..63fab8b3bb49 100644 --- a/src/google/protobuf/map_field.cc +++ b/src/google/protobuf/map_field.cc @@ -30,6 +30,7 @@ #include "google/protobuf/map_field.h" +#include #include #include "google/protobuf/port.h" @@ -43,82 +44,90 @@ namespace protobuf { namespace internal { using ::google::protobuf::internal::DownCast; +MapFieldBase::~MapFieldBase() { + ABSL_DCHECK_EQ(arena(), nullptr); + delete maybe_payload(); +} + const RepeatedPtrFieldBase& MapFieldBase::GetRepeatedField() const { ConstAccess(); SyncRepeatedFieldWithMap(); - return *reinterpret_cast(repeated_field_); + return reinterpret_cast(payload().repeated_field); } RepeatedPtrFieldBase* MapFieldBase::MutableRepeatedField() { MutableAccess(); SyncRepeatedFieldWithMap(); SetRepeatedDirty(); - return reinterpret_cast(repeated_field_); + return reinterpret_cast(&payload().repeated_field); } -void MapFieldBase::SwapState(MapFieldBase* other) { - // a relaxed swap of the atomic - auto other_state = other->state_.load(std::memory_order_relaxed); - auto this_state = state_.load(std::memory_order_relaxed); - other->state_.store(this_state, std::memory_order_relaxed); - state_.store(other_state, std::memory_order_relaxed); +template +static void SwapRelaxed(std::atomic& a, std::atomic& b) { + auto value_b = b.load(std::memory_order_relaxed); + auto value_a = a.load(std::memory_order_relaxed); + b.store(value_a, std::memory_order_relaxed); + a.store(value_b, std::memory_order_relaxed); } -void SwapRepeatedPtrToNull(RepeatedPtrField** from, - RepeatedPtrField** to, Arena* from_arena, - Arena* to_arena) { - ABSL_DCHECK(*from != nullptr); - ABSL_DCHECK(*to == nullptr); - *to = Arena::CreateMessage >(to_arena); - **to = std::move(**from); - if (from_arena == nullptr) { - delete *from; +MapFieldBase::ReflectionPayload& MapFieldBase::PayloadSlow() const { + auto p = payload_.load(std::memory_order_acquire); + if (!IsPayload(p)) { + auto* arena = ToArena(p); + auto* payload = Arena::Create(arena, arena); + auto new_p = ToTaggedPtr(payload); + if (payload_.compare_exchange_strong(p, new_p, std::memory_order_acq_rel)) { + // We were able to store it. + p = new_p; + } else { + // Someone beat us to it. Throw away the one we made. `p` already contains + // the one we want. + if (arena == nullptr) delete payload; + } } - *from = nullptr; + return *ToPayload(p); } void MapFieldBase::Swap(MapFieldBase* other) { - if (arena_ == other->arena_) { + if (arena() == other->arena()) { InternalSwap(other); return; } - if (repeated_field_ != nullptr || other->repeated_field_ != nullptr) { - if (repeated_field_ == nullptr) { - SwapRepeatedPtrToNull(&other->repeated_field_, &repeated_field_, - other->arena_, arena_); - } else if (other->repeated_field_ == nullptr) { - SwapRepeatedPtrToNull(&repeated_field_, &other->repeated_field_, arena_, - other->arena_); - } else { - repeated_field_->Swap(other->repeated_field_); - } - } - SwapState(other); + auto* p1 = maybe_payload(); + auto* p2 = other->maybe_payload(); + if (p1 == nullptr && p2 == nullptr) return; + + if (p1 == nullptr) p1 = &payload(); + if (p2 == nullptr) p2 = &other->payload(); + p1->repeated_field.Swap(&p2->repeated_field); + SwapRelaxed(p1->state, p2->state); } void MapFieldBase::UnsafeShallowSwap(MapFieldBase* other) { - ABSL_DCHECK_EQ(arena_, other->arena_); + ABSL_DCHECK_EQ(arena(), other->arena()); InternalSwap(other); } void MapFieldBase::InternalSwap(MapFieldBase* other) { - std::swap(arena_, other->arena_); - std::swap(repeated_field_, other->repeated_field_); - SwapState(other); + SwapRelaxed(payload_, other->payload_); } size_t MapFieldBase::SpaceUsedExcludingSelfLong() const { ConstAccess(); - mutex_.Lock(); - size_t size = SpaceUsedExcludingSelfNoLock(); - mutex_.Unlock(); - ConstAccess(); + size_t size = 0; + if (auto* p = maybe_payload()) { + { + absl::MutexLock lock(&p->mutex); + size = SpaceUsedExcludingSelfNoLock(); + } + ConstAccess(); + } return size; } size_t MapFieldBase::SpaceUsedExcludingSelfNoLock() const { - if (repeated_field_ != nullptr) { - return repeated_field_->SpaceUsedExcludingSelfLong(); + if (auto* p = maybe_payload()) { + return p->repeated_field.SpaceUsedExcludingSelfLong(); } else { return 0; } @@ -128,67 +137,42 @@ bool MapFieldBase::IsMapValid() const { ConstAccess(); // "Acquire" insures the operation after SyncRepeatedFieldWithMap won't get // executed before state_ is checked. - int state = state_.load(std::memory_order_acquire); - return state != STATE_MODIFIED_REPEATED; + return state() != STATE_MODIFIED_REPEATED; } bool MapFieldBase::IsRepeatedFieldValid() const { ConstAccess(); - int state = state_.load(std::memory_order_acquire); - return state != STATE_MODIFIED_MAP; + return state() != STATE_MODIFIED_MAP; } void MapFieldBase::SetMapDirty() { MutableAccess(); // These are called by (non-const) mutator functions. So by our API it's the // callers responsibility to have these calls properly ordered. - state_.store(STATE_MODIFIED_MAP, std::memory_order_relaxed); + payload().state.store(STATE_MODIFIED_MAP, std::memory_order_relaxed); } void MapFieldBase::SetRepeatedDirty() { MutableAccess(); // These are called by (non-const) mutator functions. So by our API it's the // callers responsibility to have these calls properly ordered. - state_.store(STATE_MODIFIED_REPEATED, std::memory_order_relaxed); + payload().state.store(STATE_MODIFIED_REPEATED, std::memory_order_relaxed); } void MapFieldBase::SyncRepeatedFieldWithMap() const { ConstAccess(); - // acquire here matches with release below to ensure that we can only see a - // value of CLEAN after all previous changes have been synced. - switch (state_.load(std::memory_order_acquire)) { - case STATE_MODIFIED_MAP: - mutex_.Lock(); + if (state() == STATE_MODIFIED_MAP) { + auto& p = payload(); + { + absl::MutexLock lock(&p.mutex); // Double check state, because another thread may have seen the same // state and done the synchronization before the current thread. - if (state_.load(std::memory_order_relaxed) == STATE_MODIFIED_MAP) { + if (p.state.load(std::memory_order_relaxed) == STATE_MODIFIED_MAP) { SyncRepeatedFieldWithMapNoLock(); - state_.store(CLEAN, std::memory_order_release); - } - mutex_.Unlock(); - ConstAccess(); - break; - case CLEAN: - mutex_.Lock(); - // Double check state - if (state_.load(std::memory_order_relaxed) == CLEAN) { - if (repeated_field_ == nullptr) { - repeated_field_ = - Arena::CreateMessage >(arena_); - } - state_.store(CLEAN, std::memory_order_release); + p.state.store(CLEAN, std::memory_order_release); } - mutex_.Unlock(); - ConstAccess(); - break; - default: - break; - } -} - -void MapFieldBase::SyncRepeatedFieldWithMapNoLock() const { - if (repeated_field_ == nullptr) { - repeated_field_ = Arena::CreateMessage >(arena_); + } + ConstAccess(); } } @@ -196,15 +180,17 @@ void MapFieldBase::SyncMapWithRepeatedField() const { ConstAccess(); // acquire here matches with release below to ensure that we can only see a // value of CLEAN after all previous changes have been synced. - if (state_.load(std::memory_order_acquire) == STATE_MODIFIED_REPEATED) { - mutex_.Lock(); - // Double check state, because another thread may have seen the same state - // and done the synchronization before the current thread. - if (state_.load(std::memory_order_relaxed) == STATE_MODIFIED_REPEATED) { - SyncMapWithRepeatedFieldNoLock(); - state_.store(CLEAN, std::memory_order_release); + if (state() == STATE_MODIFIED_REPEATED) { + auto& p = payload(); + { + absl::MutexLock lock(&p.mutex); + // Double check state, because another thread may have seen the same state + // and done the synchronization before the current thread. + if (p.state.load(std::memory_order_relaxed) == STATE_MODIFIED_REPEATED) { + SyncMapWithRepeatedFieldNoLock(); + p.state.store(CLEAN, std::memory_order_release); + } } - mutex_.Unlock(); ConstAccess(); } } @@ -219,7 +205,7 @@ DynamicMapField::DynamicMapField(const Message* default_entry, Arena* arena) default_entry_(default_entry) {} DynamicMapField::~DynamicMapField() { - ABSL_DCHECK_EQ(arena_, nullptr); + ABSL_DCHECK_EQ(arena(), nullptr); // DynamicMapField owns map values. Need to delete them before clearing the // map. for (auto& kv : map_) { @@ -232,7 +218,7 @@ int DynamicMapField::size() const { return GetMap().size(); } void DynamicMapField::Clear() { Map* map = &const_cast(this)->map_; - if (MapFieldBase::arena_ == nullptr) { + if (arena() == nullptr) { for (Map::iterator iter = map->begin(); iter != map->end(); ++iter) { iter->second.DeleteData(); @@ -241,8 +227,8 @@ void DynamicMapField::Clear() { map->clear(); - if (MapFieldBase::repeated_field_ != nullptr) { - MapFieldBase::repeated_field_->Clear(); + if (auto* p = maybe_payload()) { + p->repeated_field.Clear(); } // Data in map and repeated field are both empty, but we can't set status // CLEAN which will invalidate previous reference to map. @@ -261,11 +247,11 @@ void DynamicMapField::AllocateMapValue(MapValueRef* map_val) { // Allocate memory for the MapValueRef, and initialize to // default value. switch (val_des->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, TYPE) \ - case FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - TYPE* value = Arena::Create(MapFieldBase::arena_); \ - map_val->SetValue(value); \ - break; \ +#define HANDLE_TYPE(CPPTYPE, TYPE) \ + case FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + auto* value = Arena::Create(arena()); \ + map_val->SetValue(value); \ + break; \ } HANDLE_TYPE(INT32, int32_t); HANDLE_TYPE(INT64, int64_t); @@ -280,7 +266,7 @@ void DynamicMapField::AllocateMapValue(MapValueRef* map_val) { case FieldDescriptor::CPPTYPE_MESSAGE: { const Message& message = default_entry_->GetReflection()->GetMessage(*default_entry_, val_des); - Message* value = message.New(MapFieldBase::arena_); + Message* value = message.New(arena()); map_val->SetValue(value); break; } @@ -326,7 +312,7 @@ bool DynamicMapField::DeleteMapValue(const MapKey& map_key) { } // Set map dirty only if the delete is successful. MapFieldBase::SetMapDirty(); - if (MapFieldBase::arena_ == nullptr) { + if (arena() == nullptr) { iter->second.DeleteData(); } map_.erase(iter); @@ -420,31 +406,22 @@ void DynamicMapField::MergeFrom(const MapFieldBase& other) { } void DynamicMapField::Swap(MapFieldBase* other) { - DynamicMapField* other_field = DownCast(other); - std::swap(this->MapFieldBase::repeated_field_, other_field->repeated_field_); - map_.swap(other_field->map_); - // a relaxed swap of the atomic - auto other_state = other_field->state_.load(std::memory_order_relaxed); - auto this_state = this->MapFieldBase::state_.load(std::memory_order_relaxed); - other_field->state_.store(this_state, std::memory_order_relaxed); - this->MapFieldBase::state_.store(other_state, std::memory_order_relaxed); + MapFieldBase::Swap(other); + map_.swap(DownCast(other)->map_); } void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const { const Reflection* reflection = default_entry_->GetReflection(); const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key(); const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value(); - if (MapFieldBase::repeated_field_ == nullptr) { - MapFieldBase::repeated_field_ = - Arena::CreateMessage >(MapFieldBase::arena_); - } - MapFieldBase::repeated_field_->Clear(); + auto& rep = payload().repeated_field; + rep.Clear(); for (Map::const_iterator it = map_.begin(); it != map_.end(); ++it) { - Message* new_entry = default_entry_->New(MapFieldBase::arena_); - MapFieldBase::repeated_field_->AddAllocated(new_entry); + Message* new_entry = default_entry_->New(arena()); + rep.AddAllocated(new_entry); const MapKey& map_key = it->first; switch (key_des->cpp_type()) { case FieldDescriptor::CPPTYPE_STRING: @@ -515,38 +492,38 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const { const Reflection* reflection = default_entry_->GetReflection(); const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key(); const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value(); + Arena* arena = this->arena(); // DynamicMapField owns map values. Need to delete them before clearing // the map. - if (MapFieldBase::arena_ == nullptr) { + if (arena == nullptr) { for (Map::iterator iter = map->begin(); iter != map->end(); ++iter) { iter->second.DeleteData(); } } map->clear(); - for (RepeatedPtrField::iterator it = - MapFieldBase::repeated_field_->begin(); - it != MapFieldBase::repeated_field_->end(); ++it) { + auto& rep = payload().repeated_field; + for (const Message& elem : rep) { // MapKey type will be set later. MapKey map_key; switch (key_des->cpp_type()) { case FieldDescriptor::CPPTYPE_STRING: - map_key.SetStringValue(reflection->GetString(*it, key_des)); + map_key.SetStringValue(reflection->GetString(elem, key_des)); break; case FieldDescriptor::CPPTYPE_INT64: - map_key.SetInt64Value(reflection->GetInt64(*it, key_des)); + map_key.SetInt64Value(reflection->GetInt64(elem, key_des)); break; case FieldDescriptor::CPPTYPE_INT32: - map_key.SetInt32Value(reflection->GetInt32(*it, key_des)); + map_key.SetInt32Value(reflection->GetInt32(elem, key_des)); break; case FieldDescriptor::CPPTYPE_UINT64: - map_key.SetUInt64Value(reflection->GetUInt64(*it, key_des)); + map_key.SetUInt64Value(reflection->GetUInt64(elem, key_des)); break; case FieldDescriptor::CPPTYPE_UINT32: - map_key.SetUInt32Value(reflection->GetUInt32(*it, key_des)); + map_key.SetUInt32Value(reflection->GetUInt32(elem, key_des)); break; case FieldDescriptor::CPPTYPE_BOOL: - map_key.SetBoolValue(reflection->GetBool(*it, key_des)); + map_key.SetBoolValue(reflection->GetBool(elem, key_des)); break; case FieldDescriptor::CPPTYPE_DOUBLE: case FieldDescriptor::CPPTYPE_FLOAT: @@ -556,7 +533,7 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const { break; } - if (MapFieldBase::arena_ == nullptr) { + if (arena == nullptr) { // Remove existing map value with same key. Map::iterator iter = map->find(map_key); if (iter != map->end()) { @@ -567,12 +544,12 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const { MapValueRef& map_val = (*map)[map_key]; map_val.SetType(val_des->cpp_type()); switch (val_des->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, TYPE, METHOD) \ - case FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - TYPE* value = Arena::Create(MapFieldBase::arena_); \ - *value = reflection->Get##METHOD(*it, val_des); \ - map_val.SetValue(value); \ - break; \ +#define HANDLE_TYPE(CPPTYPE, TYPE, METHOD) \ + case FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + auto* value = Arena::Create(arena); \ + *value = reflection->Get##METHOD(elem, val_des); \ + map_val.SetValue(value); \ + break; \ } HANDLE_TYPE(INT32, int32_t, Int32); HANDLE_TYPE(INT64, int64_t, Int64); @@ -585,8 +562,8 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const { HANDLE_TYPE(ENUM, int32_t, EnumValue); #undef HANDLE_TYPE case FieldDescriptor::CPPTYPE_MESSAGE: { - const Message& message = reflection->GetMessage(*it, val_des); - Message* value = message.New(MapFieldBase::arena_); + const Message& message = reflection->GetMessage(elem, val_des); + Message* value = message.New(arena); value->CopyFrom(message); map_val.SetValue(value); break; @@ -597,8 +574,8 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const { size_t DynamicMapField::SpaceUsedExcludingSelfNoLock() const { size_t size = 0; - if (MapFieldBase::repeated_field_ != nullptr) { - size += MapFieldBase::repeated_field_->SpaceUsedExcludingSelfLong(); + if (auto* p = maybe_payload()) { + size += p->repeated_field.SpaceUsedExcludingSelfLong(); } size += sizeof(map_); size_t map_size = map_.size(); diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index 411911448e41..c48e7646d2c3 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -338,31 +338,14 @@ class MapFieldAccessor; // reflection implementation only. Users should never use this directly. class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { public: - MapFieldBase() - : arena_(nullptr), repeated_field_(nullptr), state_(STATE_MODIFIED_MAP) {} - - // This constructor is for constant initialized global instances. - // It uses a linker initialized mutex, so it is not compatible with regular - // runtime instances. - // Except in MSVC, where we can't have a constinit mutex. - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr MapFieldBase(ConstantInitialized) - : arena_(nullptr), - repeated_field_(nullptr), - mutex_(absl::kConstInit), - state_(STATE_MODIFIED_MAP) {} - explicit MapFieldBase(Arena* arena) - : arena_(arena), repeated_field_(nullptr), state_(STATE_MODIFIED_MAP) {} + constexpr MapFieldBase() {} + explicit MapFieldBase(Arena* arena) : payload_{ToTaggedPtr(arena)} {} MapFieldBase(const MapFieldBase&) = delete; MapFieldBase& operator=(const MapFieldBase&) = delete; protected: - ~MapFieldBase() { // "protected" stops users from deleting a `MapFieldBase *` - ABSL_DCHECK_EQ(arena_, nullptr); - delete repeated_field_; - } - void ArenaDestruct() { mutex_.~Mutex(); } - void OwnMutexDestructor(Arena& arena) { arena.OwnDestructor(&mutex_); } + // "protected" stops users from deleting a `MapFieldBase *` + ~MapFieldBase(); public: // Returns reference to internal repeated field. Data written using @@ -412,12 +395,12 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { // Synchronizes the content in Map to RepeatedPtrField if there is any change // to Map after last synchronization. void SyncRepeatedFieldWithMap() const; - virtual void SyncRepeatedFieldWithMapNoLock() const; + virtual void SyncRepeatedFieldWithMapNoLock() const = 0; // Synchronizes the content in RepeatedPtrField to Map if there is any change // to RepeatedPtrField after last synchronization. void SyncMapWithRepeatedField() const; - virtual void SyncMapWithRepeatedFieldNoLock() const {} + virtual void SyncMapWithRepeatedFieldNoLock() const = 0; // Tells MapFieldBase that there is new change to Map. void SetMapDirty(); @@ -457,12 +440,39 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { CLEAN = 2, // data in map and repeated field are same }; - Arena* arena_; - mutable RepeatedPtrField* repeated_field_; + struct ReflectionPayload { + explicit ReflectionPayload(Arena* arena) : repeated_field(arena) {} + RepeatedPtrField repeated_field; + + absl::Mutex mutex; // The thread to synchronize map and repeated + // field needs to get lock first; + std::atomic state{STATE_MODIFIED_MAP}; + }; + + Arena* arena() const { + auto p = payload_.load(std::memory_order_acquire); + if (IsPayload(p)) return ToPayload(p)->repeated_field.GetArena(); + return ToArena(p); + } + + // Returns the reflection payload. Returns null if it does not exist yet. + ReflectionPayload* maybe_payload() const { + auto p = payload_.load(std::memory_order_acquire); + return IsPayload(p) ? ToPayload(p) : nullptr; + } + // Returns the reflection payload, and constructs one if does not exist yet. + ReflectionPayload& payload() const { + auto* p = maybe_payload(); + return p != nullptr ? *p : PayloadSlow(); + } + ReflectionPayload& PayloadSlow() const; - mutable absl::Mutex mutex_; // The thread to synchronize map and repeated - // field needs to get lock first; - mutable std::atomic state_; + State state() const { + auto* p = maybe_payload(); + return p != nullptr ? p->state.load(std::memory_order_acquire) + // The default + : STATE_MODIFIED_MAP; + } private: friend class ContendedMapCleanTest; @@ -492,8 +502,32 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { // It implements the ++ operator of MapIterator. virtual void IncreaseIterator(MapIterator* map_iter) const = 0; - // Swaps state_ with another MapFieldBase - void SwapState(MapFieldBase* other); + enum class TaggedPtr : uintptr_t {}; + static constexpr uintptr_t kHasPayloadBit = 1; + + static ReflectionPayload* ToPayload(TaggedPtr p) { + ABSL_DCHECK(IsPayload(p)); + auto* res = reinterpret_cast(static_cast(p) - + kHasPayloadBit); + PROTOBUF_ASSUME(res != nullptr); + return res; + } + static Arena* ToArena(TaggedPtr p) { + ABSL_DCHECK(!IsPayload(p)); + return reinterpret_cast(p); + } + static TaggedPtr ToTaggedPtr(ReflectionPayload* p) { + return static_cast(reinterpret_cast(p) + + kHasPayloadBit); + } + static TaggedPtr ToTaggedPtr(Arena* p) { + return static_cast(reinterpret_cast(p)); + } + static bool IsPayload(TaggedPtr p) { + return static_cast(p) & kHasPayloadBit; + } + + mutable std::atomic payload_{}; }; // This class provides common Map Reflection implementations for generated @@ -501,28 +535,21 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { template class TypeDefinedMapFieldBase : public MapFieldBase { public: - TypeDefinedMapFieldBase() {} + constexpr TypeDefinedMapFieldBase() {} TypeDefinedMapFieldBase(const TypeDefinedMapFieldBase&) = delete; TypeDefinedMapFieldBase& operator=(const TypeDefinedMapFieldBase&) = delete; - // This constructor is for constant initialized global instances. - // It uses a linker initialized mutex, so it is not compatible with regular - // runtime instances. - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr TypeDefinedMapFieldBase(ConstantInitialized tag) - : MapFieldBase(tag) {} explicit TypeDefinedMapFieldBase(Arena* arena) : MapFieldBase(arena) {} TypeDefinedMapFieldBase(ArenaInitialized, Arena* arena) : TypeDefinedMapFieldBase(arena) {} protected: ~TypeDefinedMapFieldBase() {} - using MapFieldBase::ArenaDestruct; public: - void MapBegin(MapIterator* map_iter) const override; - void MapEnd(MapIterator* map_iter) const override; - bool EqualIterator(const MapIterator& a, const MapIterator& b) const override; + void MapBegin(MapIterator* map_iter) const final; + void MapEnd(MapIterator* map_iter) const final; + bool EqualIterator(const MapIterator& a, const MapIterator& b) const final; virtual const Map& GetMap() const = 0; // This overrides the base's method to specialize the signature via @@ -534,11 +561,11 @@ class TypeDefinedMapFieldBase : public MapFieldBase { const MapIterator* map_iter) const; private: - void InitializeIterator(MapIterator* map_iter) const override; - void DeleteIterator(MapIterator* map_iter) const override; + void InitializeIterator(MapIterator* map_iter) const final; + void DeleteIterator(MapIterator* map_iter) const final; void CopyIterator(MapIterator* this_iteratorm, - const MapIterator& that_iterator) const override; - void IncreaseIterator(MapIterator* map_iter) const override; + const MapIterator& that_iterator) const final; + void IncreaseIterator(MapIterator* map_iter) const final; virtual void SetMapIteratorValue(MapIterator* map_iter) const = 0; }; @@ -549,7 +576,7 @@ class TypeDefinedMapFieldBase : public MapFieldBase { template -class MapField : public TypeDefinedMapFieldBase { +class MapField final : public TypeDefinedMapFieldBase { // Provide utilities to parse/serialize key/value. Provide utilities to // manipulate internal stored type. typedef MapTypeHandler KeyTypeHandler; @@ -574,47 +601,40 @@ class MapField : public TypeDefinedMapFieldBase { static constexpr WireFormatLite::FieldType kKeyFieldType = kKeyFieldType_; static constexpr WireFormatLite::FieldType kValueFieldType = kValueFieldType_; - MapField() : impl_() {} + constexpr MapField() : impl_() {} MapField(const MapField&) = delete; MapField& operator=(const MapField&) = delete; - virtual ~MapField() = default; - void ArenaDestruct() { TypeDefinedMapFieldBase::ArenaDestruct(); } - - // This constructor is for constant initialized global instances. - // It uses a linker initialized mutex, so it is not compatible with regular - // runtime instances. - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr MapField(ConstantInitialized tag) - : TypeDefinedMapFieldBase(tag), impl_() {} + ~MapField() = default; + explicit MapField(Arena* arena) : TypeDefinedMapFieldBase(arena), impl_(arena) {} MapField(ArenaInitialized, Arena* arena) : MapField(arena) {} + // TODO(sbenza): Move these up to TypeDefinedMapFieldBase where possible. // Implement MapFieldBase - bool ContainsMapKey(const MapKey& map_key) const override; - bool InsertOrLookupMapValue(const MapKey& map_key, MapValueRef* val) override; - bool LookupMapValue(const MapKey& map_key, - MapValueConstRef* val) const override; + bool ContainsMapKey(const MapKey& map_key) const final; + bool InsertOrLookupMapValue(const MapKey& map_key, MapValueRef* val) final; + bool LookupMapValue(const MapKey& map_key, MapValueConstRef* val) const final; bool LookupMapValue(const MapKey&, MapValueRef*) const = delete; - bool DeleteMapValue(const MapKey& map_key) override; + bool DeleteMapValue(const MapKey& map_key) final; - const Map& GetMap() const override { + const Map& GetMap() const final { MapFieldBase::SyncMapWithRepeatedField(); return impl_.GetMap(); } - Map* MutableMap() override { + Map* MutableMap() final { MapFieldBase::SyncMapWithRepeatedField(); Map* result = impl_.MutableMap(); MapFieldBase::SetMapDirty(); return result; } - int size() const override; - void Clear() override; - void MergeFrom(const MapFieldBase& other) override; - void Swap(MapFieldBase* other) override; - void UnsafeShallowSwap(MapFieldBase* other) override; + int size() const final; + void Clear() final; + void MergeFrom(const MapFieldBase& other) final; + void Swap(MapFieldBase* other) final; + void UnsafeShallowSwap(MapFieldBase* other) final; void InternalSwap(MapField* other); // Used in the implementation of parsing. Caller should take the ownership iff @@ -639,25 +659,20 @@ class MapField : public TypeDefinedMapFieldBase { typedef void DestructorSkippable_; // Implements MapFieldBase - void SyncRepeatedFieldWithMapNoLock() const override; - void SyncMapWithRepeatedFieldNoLock() const override; - size_t SpaceUsedExcludingSelfNoLock() const override; + void SyncRepeatedFieldWithMapNoLock() const final; + void SyncMapWithRepeatedFieldNoLock() const final; + size_t SpaceUsedExcludingSelfNoLock() const final; - void SetMapIteratorValue(MapIterator* map_iter) const override; + void SetMapIteratorValue(MapIterator* map_iter) const final; friend class ::PROTOBUF_NAMESPACE_ID::Arena; friend class MapFieldStateTest; // For testing, it needs raw access to impl_ }; -template -bool AllAreInitialized( - const MapField& field) { - const auto& t = field.GetMap(); - for (typename Map::const_iterator it = t.begin(); it != t.end(); - ++it) { - if (!it->second.IsInitialized()) return false; +template +bool AllAreInitialized(const TypeDefinedMapFieldBase& field) { + for (const auto& p : field.GetMap()) { + if (!p.second.IsInitialized()) return false; } return true; } @@ -670,7 +685,7 @@ struct MapEntryToMapField< typedef MapField MapFieldType; }; -class PROTOBUF_EXPORT DynamicMapField +class PROTOBUF_EXPORT DynamicMapField final : public TypeDefinedMapFieldBase { public: explicit DynamicMapField(const Message* default_entry); @@ -680,21 +695,20 @@ class PROTOBUF_EXPORT DynamicMapField virtual ~DynamicMapField(); // Implement MapFieldBase - bool ContainsMapKey(const MapKey& map_key) const override; - bool InsertOrLookupMapValue(const MapKey& map_key, MapValueRef* val) override; - bool LookupMapValue(const MapKey& map_key, - MapValueConstRef* val) const override; + bool ContainsMapKey(const MapKey& map_key) const final; + bool InsertOrLookupMapValue(const MapKey& map_key, MapValueRef* val) final; + bool LookupMapValue(const MapKey& map_key, MapValueConstRef* val) const final; bool LookupMapValue(const MapKey&, MapValueRef*) const = delete; - bool DeleteMapValue(const MapKey& map_key) override; - void MergeFrom(const MapFieldBase& other) override; - void Swap(MapFieldBase* other) override; - void UnsafeShallowSwap(MapFieldBase* other) override { Swap(other); } + bool DeleteMapValue(const MapKey& map_key) final; + void MergeFrom(const MapFieldBase& other) final; + void Swap(MapFieldBase* other) final; + void UnsafeShallowSwap(MapFieldBase* other) final { Swap(other); } - const Map& GetMap() const override; - Map* MutableMap() override; + const Map& GetMap() const final; + Map* MutableMap() final; - int size() const override; - void Clear() override; + int size() const final; + void Clear() final; private: Map map_; @@ -703,10 +717,10 @@ class PROTOBUF_EXPORT DynamicMapField void AllocateMapValue(MapValueRef* map_val); // Implements MapFieldBase - void SyncRepeatedFieldWithMapNoLock() const override; - void SyncMapWithRepeatedFieldNoLock() const override; - size_t SpaceUsedExcludingSelfNoLock() const override; - void SetMapIteratorValue(MapIterator* map_iter) const override; + void SyncRepeatedFieldWithMapNoLock() const final; + void SyncMapWithRepeatedFieldNoLock() const final; + size_t SpaceUsedExcludingSelfNoLock() const final; + void SetMapIteratorValue(MapIterator* map_iter) const final; }; } // namespace internal diff --git a/src/google/protobuf/map_field_inl.h b/src/google/protobuf/map_field_inl.h index 677ccc9bc42e..155b8e045451 100644 --- a/src/google/protobuf/map_field_inl.h +++ b/src/google/protobuf/map_field_inl.h @@ -178,11 +178,8 @@ template void MapField::Clear() { - if (this->MapFieldBase::repeated_field_ != nullptr) { - RepeatedPtrField* repeated_field = - reinterpret_cast*>( - this->MapFieldBase::repeated_field_); - repeated_field->Clear(); + if (auto* p = this->maybe_payload()) { + reinterpret_cast&>(p->repeated_field).Clear(); } impl_.MutableMap()->clear(); @@ -308,15 +305,10 @@ template void MapField::SyncRepeatedFieldWithMapNoLock() const { - if (this->MapFieldBase::repeated_field_ == nullptr) { - this->MapFieldBase::repeated_field_ = - Arena::CreateMessage >( - this->MapFieldBase::arena_); - } const Map& map = impl_.GetMap(); RepeatedPtrField* repeated_field = reinterpret_cast*>( - this->MapFieldBase::repeated_field_); + &this->payload().repeated_field); repeated_field->Clear(); @@ -325,11 +317,11 @@ void MapFieldarena(); const Message* default_entry = Derived::internal_default_instance(); for (typename Map::const_iterator it = map.begin(); it != map.end(); ++it) { - EntryType* new_entry = - DownCast(default_entry->New(this->MapFieldBase::arena_)); + EntryType* new_entry = DownCast(default_entry->New(arena)); repeated_field->AddAllocated(new_entry); (*new_entry->mutable_key()) = it->first; (*new_entry->mutable_value()) = it->second; @@ -344,8 +336,7 @@ void MapField* map = const_cast(this)->impl_.MutableMap(); RepeatedPtrField* repeated_field = reinterpret_cast*>( - this->MapFieldBase::repeated_field_); - ABSL_CHECK(this->MapFieldBase::repeated_field_ != nullptr); + &this->payload().repeated_field); map->clear(); for (typename RepeatedPtrField::iterator it = repeated_field->begin(); @@ -365,8 +356,8 @@ template ::SpaceUsedExcludingSelfNoLock() const { size_t size = 0; - if (this->MapFieldBase::repeated_field_ != nullptr) { - size += this->MapFieldBase::repeated_field_->SpaceUsedExcludingSelfLong(); + if (auto* p = this->maybe_payload()) { + size += p->repeated_field.SpaceUsedExcludingSelfLong(); } size += impl_.GetMap().SpaceUsedExcludingSelfLong(); diff --git a/src/google/protobuf/map_field_test.cc b/src/google/protobuf/map_field_test.cc index 74b783526174..ae6007ffe050 100644 --- a/src/google/protobuf/map_field_test.cc +++ b/src/google/protobuf/map_field_test.cc @@ -54,33 +54,21 @@ namespace internal { using unittest::TestAllTypes; -// ArenaHolder from map_test_util.h works fine for fields other than map -// fields. For arena-owned map fields, the ArenaDestruct() call must be made -// because the destructor will be skipped. -template -struct ArenaDestructor : ArenaHolder { - ArenaDestructor(Arena* arena) - : ArenaHolder(arena), owned_by_arena(arena != nullptr) {} - ~ArenaDestructor() { - if (owned_by_arena) ArenaHolder::get()->ArenaDestruct(); - } - - bool owned_by_arena; -}; - class MapFieldBaseStub : public MapFieldBase { public: - typedef void InternalArenaConstructable_; + using InternalArenaConstructable_ = void; typedef void DestructorSkippable_; MapFieldBaseStub() {} virtual ~MapFieldBaseStub() {} explicit MapFieldBaseStub(Arena* arena) : MapFieldBase(arena) {} void SetMapDirty() { - state_.store(STATE_MODIFIED_MAP, std::memory_order_relaxed); + payload().state.store(STATE_MODIFIED_MAP, std::memory_order_relaxed); } void SetRepeatedDirty() { - state_.store(STATE_MODIFIED_REPEATED, std::memory_order_relaxed); + payload().state.store(STATE_MODIFIED_REPEATED, std::memory_order_relaxed); } + void SyncRepeatedFieldWithMapNoLock() const override {} + void SyncMapWithRepeatedFieldNoLock() const override {} UntypedMapBase* MutableMap() override { return nullptr; } bool ContainsMapKey(const MapKey& map_key) const override { return false; } bool InsertOrLookupMapValue(const MapKey& map_key, @@ -142,7 +130,7 @@ class MapFieldBasePrimitiveTest : public testing::TestWithParam { } std::unique_ptr arena_; - ArenaDestructor map_field_; + ArenaHolder map_field_; MapFieldBase* map_field_base_; Map* map_; const Descriptor* map_descriptor_; @@ -246,7 +234,7 @@ class MapFieldStateTest map_field_base_(map_field_.get()), state_(std::get<0>(GetParam())) { // Build map field - Expect(map_field_.get(), MAP_DIRTY, 0, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 0, 0); switch (state_) { case CLEAN: AddOneStillClean(map_field_.get()); @@ -267,13 +255,13 @@ class MapFieldStateTest Map* map = map_field->MutableMap(); (*map)[0] = 0; map_field_base->GetRepeatedField(); - Expect(map_field, CLEAN, 1, 1, false); + Expect(map_field, CLEAN, 1, 1); } void MakeMapDirty(MapFieldType* map_field) { Map* map = map_field->MutableMap(); (*map)[0] = 0; - Expect(map_field, MAP_DIRTY, 1, 0, true); + Expect(map_field, MAP_DIRTY, 1, 0); } void MakeRepeatedDirty(MapFieldType* map_field) { @@ -284,32 +272,28 @@ class MapFieldStateTest Map* map = map_field->impl_.MutableMap(); map->clear(); - Expect(map_field, REPEATED_DIRTY, 0, 1, false); + Expect(map_field, REPEATED_DIRTY, 0, 1); } void Expect(MapFieldType* map_field, State state, int map_size, - int repeated_size, bool is_repeated_null) { + int repeated_size) { // We use MutableMap on impl_ because we don't want to disturb the syncing Map* map = map_field->impl_.MutableMap(); - RepeatedPtrField* repeated_field = map_field->repeated_field_; switch (state) { case MAP_DIRTY: - EXPECT_FALSE(map_field->state_.load(std::memory_order_relaxed) != - MapFieldType::STATE_MODIFIED_MAP); - EXPECT_TRUE(map_field->state_.load(std::memory_order_relaxed) != + EXPECT_FALSE(map_field->state() != MapFieldType::STATE_MODIFIED_MAP); + EXPECT_TRUE(map_field->state() != MapFieldType::STATE_MODIFIED_REPEATED); break; case REPEATED_DIRTY: - EXPECT_TRUE(map_field->state_.load(std::memory_order_relaxed) != - MapFieldType::STATE_MODIFIED_MAP); - EXPECT_FALSE(map_field->state_.load(std::memory_order_relaxed) != + EXPECT_TRUE(map_field->state() != MapFieldType::STATE_MODIFIED_MAP); + EXPECT_FALSE(map_field->state() != MapFieldType::STATE_MODIFIED_REPEATED); break; case CLEAN: - EXPECT_TRUE(map_field->state_.load(std::memory_order_relaxed) != - MapFieldType::STATE_MODIFIED_MAP); - EXPECT_TRUE(map_field->state_.load(std::memory_order_relaxed) != + EXPECT_TRUE(map_field->state() != MapFieldType::STATE_MODIFIED_MAP); + EXPECT_TRUE(map_field->state() != MapFieldType::STATE_MODIFIED_REPEATED); break; default: @@ -317,19 +301,14 @@ class MapFieldStateTest } EXPECT_EQ(map_size, map->size()); - if (is_repeated_null) { - EXPECT_TRUE(repeated_field == nullptr); - } else { - if (repeated_field == nullptr) { - EXPECT_EQ(repeated_size, 0); - } else { - EXPECT_EQ(repeated_size, repeated_field->size()); - } - } + EXPECT_EQ(repeated_size, + map_field->maybe_payload() == nullptr + ? 0 + : map_field->maybe_payload()->repeated_field.size()); } std::unique_ptr arena_; - ArenaDestructor map_field_; + ArenaHolder map_field_; MapFieldBase* map_field_base_; State state_; }; @@ -342,83 +321,83 @@ INSTANTIATE_TEST_SUITE_P(MapFieldStateTestInstance, MapFieldStateTest, TEST_P(MapFieldStateTest, GetMap) { map_field_->GetMap(); if (state_ != MAP_DIRTY) { - Expect(map_field_.get(), CLEAN, 1, 1, false); + Expect(map_field_.get(), CLEAN, 1, 1); } else { - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); } } TEST_P(MapFieldStateTest, MutableMap) { map_field_->MutableMap(); if (state_ != MAP_DIRTY) { - Expect(map_field_.get(), MAP_DIRTY, 1, 1, false); + Expect(map_field_.get(), MAP_DIRTY, 1, 1); } else { - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); } } TEST_P(MapFieldStateTest, MergeFromClean) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); AddOneStillClean(other.get()); map_field_->MergeFrom(*other); if (state_ != MAP_DIRTY) { - Expect(map_field_.get(), MAP_DIRTY, 1, 1, false); + Expect(map_field_.get(), MAP_DIRTY, 1, 1); } else { - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); } - Expect(other.get(), CLEAN, 1, 1, false); + Expect(other.get(), CLEAN, 1, 1); } TEST_P(MapFieldStateTest, MergeFromMapDirty) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); MakeMapDirty(other.get()); map_field_->MergeFrom(*other); if (state_ != MAP_DIRTY) { - Expect(map_field_.get(), MAP_DIRTY, 1, 1, false); + Expect(map_field_.get(), MAP_DIRTY, 1, 1); } else { - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); } - Expect(other.get(), MAP_DIRTY, 1, 0, true); + Expect(other.get(), MAP_DIRTY, 1, 0); } TEST_P(MapFieldStateTest, MergeFromRepeatedDirty) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); MakeRepeatedDirty(other.get()); map_field_->MergeFrom(*other); if (state_ != MAP_DIRTY) { - Expect(map_field_.get(), MAP_DIRTY, 1, 1, false); + Expect(map_field_.get(), MAP_DIRTY, 1, 1); } else { - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); } - Expect(other.get(), CLEAN, 1, 1, false); + Expect(other.get(), CLEAN, 1, 1); } TEST_P(MapFieldStateTest, SwapClean) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); AddOneStillClean(other.get()); map_field_->Swap(other.get()); - Expect(map_field_.get(), CLEAN, 1, 1, false); + Expect(map_field_.get(), CLEAN, 1, 1); switch (state_) { case CLEAN: - Expect(other.get(), CLEAN, 1, 1, false); + Expect(other.get(), CLEAN, 1, 1); break; case MAP_DIRTY: - Expect(other.get(), MAP_DIRTY, 1, 0, true); + Expect(other.get(), MAP_DIRTY, 1, 0); break; case REPEATED_DIRTY: - Expect(other.get(), REPEATED_DIRTY, 0, 1, false); + Expect(other.get(), REPEATED_DIRTY, 0, 1); break; default: break; @@ -426,22 +405,22 @@ TEST_P(MapFieldStateTest, SwapClean) { } TEST_P(MapFieldStateTest, SwapMapDirty) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); MakeMapDirty(other.get()); map_field_->Swap(other.get()); - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); switch (state_) { case CLEAN: - Expect(other.get(), CLEAN, 1, 1, false); + Expect(other.get(), CLEAN, 1, 1); break; case MAP_DIRTY: - Expect(other.get(), MAP_DIRTY, 1, 0, true); + Expect(other.get(), MAP_DIRTY, 1, 0); break; case REPEATED_DIRTY: - Expect(other.get(), REPEATED_DIRTY, 0, 1, false); + Expect(other.get(), REPEATED_DIRTY, 0, 1); break; default: break; @@ -449,22 +428,22 @@ TEST_P(MapFieldStateTest, SwapMapDirty) { } TEST_P(MapFieldStateTest, SwapRepeatedDirty) { - ArenaDestructor other(arena_.get()); + ArenaHolder other(arena_.get()); MakeRepeatedDirty(other.get()); map_field_->Swap(other.get()); - Expect(map_field_.get(), REPEATED_DIRTY, 0, 1, false); + Expect(map_field_.get(), REPEATED_DIRTY, 0, 1); switch (state_) { case CLEAN: - Expect(other.get(), CLEAN, 1, 1, false); + Expect(other.get(), CLEAN, 1, 1); break; case MAP_DIRTY: - Expect(other.get(), MAP_DIRTY, 1, 0, true); + Expect(other.get(), MAP_DIRTY, 1, 0); break; case REPEATED_DIRTY: - Expect(other.get(), REPEATED_DIRTY, 0, 1, false); + Expect(other.get(), REPEATED_DIRTY, 0, 1); break; default: break; @@ -474,7 +453,7 @@ TEST_P(MapFieldStateTest, SwapRepeatedDirty) { TEST_P(MapFieldStateTest, Clear) { map_field_->Clear(); - Expect(map_field_.get(), MAP_DIRTY, 0, 0, false); + Expect(map_field_.get(), MAP_DIRTY, 0, 0); } TEST_P(MapFieldStateTest, SpaceUsedExcludingSelf) { @@ -482,13 +461,13 @@ TEST_P(MapFieldStateTest, SpaceUsedExcludingSelf) { switch (state_) { case CLEAN: - Expect(map_field_.get(), CLEAN, 1, 1, false); + Expect(map_field_.get(), CLEAN, 1, 1); break; case MAP_DIRTY: - Expect(map_field_.get(), MAP_DIRTY, 1, 0, true); + Expect(map_field_.get(), MAP_DIRTY, 1, 0); break; case REPEATED_DIRTY: - Expect(map_field_.get(), REPEATED_DIRTY, 0, 1, false); + Expect(map_field_.get(), REPEATED_DIRTY, 0, 1); break; default: break; @@ -499,9 +478,9 @@ TEST_P(MapFieldStateTest, GetMapField) { map_field_base_->GetRepeatedField(); if (state_ != REPEATED_DIRTY) { - Expect(map_field_.get(), CLEAN, 1, 1, false); + Expect(map_field_.get(), CLEAN, 1, 1); } else { - Expect(map_field_.get(), REPEATED_DIRTY, 0, 1, false); + Expect(map_field_.get(), REPEATED_DIRTY, 0, 1); } } @@ -509,20 +488,16 @@ TEST_P(MapFieldStateTest, MutableMapField) { map_field_base_->MutableRepeatedField(); if (state_ != REPEATED_DIRTY) { - Expect(map_field_.get(), REPEATED_DIRTY, 1, 1, false); + Expect(map_field_.get(), REPEATED_DIRTY, 1, 1); } else { - Expect(map_field_.get(), REPEATED_DIRTY, 0, 1, false); + Expect(map_field_.get(), REPEATED_DIRTY, 0, 1); } } -class MyMapField - : public MapField { - public: - constexpr MyMapField() - : MyMapField::MapField(internal::ConstantInitialized{}) {} -}; +using MyMapField = + MapField; TEST(MapFieldTest, ConstInit) { // This tests that `MapField` and all its base classes can be constant From 132c67c2c75cafb349515cd51bdf0bad07ee1510 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 25 Apr 2023 20:14:22 +0000 Subject: [PATCH 424/463] Auto-generate files after cl/527051764 --- src/google/protobuf/struct.pb.cc | 10 +--------- src/google/protobuf/struct.pb.h | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index e41a61416612..f28c025e23f5 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -33,7 +33,7 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 Struct_FieldsEntry_DoNotUseDefaultTypeInternal _Struct_FieldsEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR Struct::Struct( ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_.fields_)*/{::_pbi::ConstantInitialized()} + /*decltype(_impl_.fields_)*/{} , /*decltype(_impl_._cached_size_)*/{}} {} struct StructDefaultTypeInternal { PROTOBUF_CONSTEXPR StructDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} @@ -231,9 +231,6 @@ class Struct::_Internal { Struct::Struct(::PROTOBUF_NAMESPACE_ID::Arena* arena) : ::PROTOBUF_NAMESPACE_ID::Message(arena) { SharedCtor(arena); - if (arena != nullptr) { - arena->OwnCustomDestructor(this, &Struct::ArenaDtor); - } // @@protoc_insertion_point(arena_constructor:google.protobuf.Struct) } Struct::Struct(const Struct& from) @@ -260,7 +257,6 @@ Struct::~Struct() { // @@protoc_insertion_point(destructor:google.protobuf.Struct) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; - ArenaDtor(this); return; } SharedDtor(); @@ -271,10 +267,6 @@ inline void Struct::SharedDtor() { _impl_.fields_.~MapField(); } -void Struct::ArenaDtor(void* object) { - Struct* _this = reinterpret_cast< Struct* >(object); - _this->_impl_.fields_.ArenaDestruct(); -} void Struct::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 9752c2325f90..ff46f9968e83 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -255,8 +255,6 @@ class PROTOBUF_EXPORT Struct final : } protected: explicit Struct(::PROTOBUF_NAMESPACE_ID::Arena* arena); - private: - static void ArenaDtor(void* object); public: static const ClassData _class_data_; From b26afb5a86cd9f764e35795c41c0f94a012db19e Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Tue, 25 Apr 2023 13:37:18 -0700 Subject: [PATCH 425/463] Update protobuf's UPB dep in preparation for the 23.0-rc1 release. PiperOrigin-RevId: 527061210 --- php/ext/google/protobuf/php-upb.c | 2 +- php/ext/google/protobuf/php-upb.h | 653 +++++++++++++++++++++++++- protobuf_deps.bzl | 4 +- ruby/ext/google/protobuf_c/ruby-upb.c | 2 +- ruby/ext/google/protobuf_c/ruby-upb.h | 653 +++++++++++++++++++++++++- 5 files changed, 1304 insertions(+), 10 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 9d52c235c3a0..51d3d4257e12 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -7225,7 +7225,7 @@ bool upb_MiniTable_Link(upb_MiniTable* mt, const upb_MiniTable** sub_tables, upb_MiniTableField* f = (upb_MiniTableField*)&mt->fields[i]; if (upb_MiniTableField_CType(f) == kUpb_CType_Enum) { const upb_MiniTableEnum* sub = sub_enums[enum_count++]; - if (enum_count > sub_table_count) return false; + if (enum_count > sub_enum_count) return false; if (sub != NULL) { if (!upb_MiniTable_SetSubEnum(mt, f, sub)) return false; } diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 793037f43c7d..3aa0454c6ece 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -365,7 +365,7 @@ typedef enum { kUpb_CType_Float = 2, kUpb_CType_Int32 = 3, kUpb_CType_UInt32 = 4, - kUpb_CType_Enum = 5, // Enum values are int32. + kUpb_CType_Enum = 5, // Enum values are int32. TODO(b/279178239): rename kUpb_CType_Message = 6, kUpb_CType_Double = 7, kUpb_CType_Int64 = 8, @@ -2720,7 +2720,8 @@ UPB_API_INLINE int32_t upb_Message_GetInt32(const upb_Message* msg, UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, const upb_MiniTableField* field, int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32); + UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || + upb_MiniTableField_CType(field) == kUpb_CType_Enum); UPB_ASSUME(_upb_MiniTableField_GetRep(field) == kUpb_FieldRep_4Byte); UPB_ASSUME(!upb_IsRepeatedOrMap(field)); return _upb_Message_SetField(msg, field, &value, a); @@ -2746,7 +2747,7 @@ UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, return _upb_Message_SetField(msg, field, &value, a); } -UPB_API_INLINE void upb_Message_SetEnumProto2( +UPB_API_INLINE void upb_Message_SetClosedEnum( upb_Message* msg, const upb_MiniTable* msg_mini_table, const upb_MiniTableField* field, int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(field)); @@ -3472,6 +3473,23 @@ UPB_INLINE const google_protobuf_FileDescriptorProto* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorSet_file_upb_array(const google_protobuf_FileDescriptorSet* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorSet_file_mutable_upb_array(const google_protobuf_FileDescriptorSet* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorSet_has_file(const google_protobuf_FileDescriptorSet* msg) { size_t size; google_protobuf_FileDescriptorSet_file(msg, &size); @@ -3585,6 +3603,23 @@ UPB_INLINE upb_StringView const* google_protobuf_FileDescriptorProto_dependency( return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {3, UPB_SIZE(4, 40), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {3, UPB_SIZE(4, 40), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_dependency(msg, &size); @@ -3605,6 +3640,23 @@ UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_FileDes return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_message_type_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(8, 48), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_message_type_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(8, 48), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_message_type(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_message_type(msg, &size); @@ -3625,6 +3677,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_enum_type_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(12, 56), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_enum_type_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(12, 56), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_enum_type(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_enum_type(msg, &size); @@ -3645,6 +3714,23 @@ UPB_INLINE const google_protobuf_ServiceDescriptorProto* const* google_protobuf_ return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_service_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(16, 64), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_service_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(16, 64), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_service(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_service(msg, &size); @@ -3665,6 +3751,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_Fi return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_extension_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {7, UPB_SIZE(20, 72), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_extension_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {7, UPB_SIZE(20, 72), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_extension(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_extension(msg, &size); @@ -3715,6 +3818,23 @@ UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_public_dependency( return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_public_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {10, UPB_SIZE(32, 96), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_public_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {10, UPB_SIZE(32, 96), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_public_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_public_dependency(msg, &size); @@ -3735,6 +3855,23 @@ UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_weak_dependency(co return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_weak_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {11, UPB_SIZE(36, 104), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_weak_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {11, UPB_SIZE(36, 104), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_weak_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_weak_dependency(msg, &size); @@ -4053,6 +4190,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_field_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_field_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_field(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_field(msg, &size); @@ -4073,6 +4227,23 @@ UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_Descrip return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_nested_type_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {3, UPB_SIZE(8, 32), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_nested_type_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {3, UPB_SIZE(8, 32), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_nested_type(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_nested_type(msg, &size); @@ -4093,6 +4264,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_Des return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_enum_type_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_enum_type_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_enum_type(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_enum_type(msg, &size); @@ -4113,6 +4301,23 @@ UPB_INLINE const google_protobuf_DescriptorProto_ExtensionRange* const* google_p return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_extension_range_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_extension_range_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_extension_range(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_extension_range(msg, &size); @@ -4133,6 +4338,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_extension_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(20, 56), 0, 4, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_extension_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(20, 56), 0, 4, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_extension(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_extension(msg, &size); @@ -4168,6 +4390,23 @@ UPB_INLINE const google_protobuf_OneofDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_oneof_decl_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {8, UPB_SIZE(28, 72), 0, 6, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_oneof_decl_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {8, UPB_SIZE(28, 72), 0, 6, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_oneof_decl(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_oneof_decl(msg, &size); @@ -4188,6 +4427,23 @@ UPB_INLINE const google_protobuf_DescriptorProto_ReservedRange* const* google_pr return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_reserved_range_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {9, UPB_SIZE(32, 80), 0, 7, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_reserved_range_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {9, UPB_SIZE(32, 80), 0, 7, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_reserved_range(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_reserved_range(msg, &size); @@ -4208,6 +4464,23 @@ UPB_INLINE upb_StringView const* google_protobuf_DescriptorProto_reserved_name(c return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_reserved_name_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {10, UPB_SIZE(36, 88), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_reserved_name_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {10, UPB_SIZE(36, 88), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_reserved_name(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_reserved_name(msg, &size); @@ -4664,6 +4937,23 @@ UPB_INLINE const google_protobuf_ExtensionRangeOptions_Declaration* const* googl return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ExtensionRangeOptions_declaration_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ExtensionRangeOptions_declaration_mutable_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ExtensionRangeOptions_has_declaration(const google_protobuf_ExtensionRangeOptions* msg) { size_t size; google_protobuf_ExtensionRangeOptions_declaration(msg, &size); @@ -4699,6 +4989,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Ext return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ExtensionRangeOptions_uninterpreted_option_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(12, 16), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ExtensionRangeOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(12, 16), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ExtensionRangeOptions_has_uninterpreted_option(const google_protobuf_ExtensionRangeOptions* msg) { size_t size; google_protobuf_ExtensionRangeOptions_uninterpreted_option(msg, &size); @@ -5315,6 +5622,23 @@ UPB_INLINE const google_protobuf_EnumValueDescriptorProto* const* google_protobu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_value_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_value_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_value(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_value(msg, &size); @@ -5350,6 +5674,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto_EnumReservedRange* const* g return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_reserved_range_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_reserved_range_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_reserved_range(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_reserved_range(msg, &size); @@ -5370,6 +5711,23 @@ UPB_INLINE upb_StringView const* google_protobuf_EnumDescriptorProto_reserved_na return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_reserved_name_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_reserved_name_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_reserved_name(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_reserved_name(msg, &size); @@ -5711,6 +6069,23 @@ UPB_INLINE const google_protobuf_MethodDescriptorProto* const* google_protobuf_S return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ServiceDescriptorProto_method_upb_array(const google_protobuf_ServiceDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ServiceDescriptorProto_method_mutable_upb_array(const google_protobuf_ServiceDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_method(const google_protobuf_ServiceDescriptorProto* msg) { size_t size; google_protobuf_ServiceDescriptorProto_method(msg, &size); @@ -6284,6 +6659,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileOptions_uninterpreted_option_upb_array(const google_protobuf_FileOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(20, 184), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_FileOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(20, 184), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileOptions_has_uninterpreted_option(const google_protobuf_FileOptions* msg) { size_t size; google_protobuf_FileOptions_uninterpreted_option(msg, &size); @@ -6522,6 +6914,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Mes return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_MessageOptions_uninterpreted_option_upb_array(const google_protobuf_MessageOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 8, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_MessageOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_MessageOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 8, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_MessageOptions_has_uninterpreted_option(const google_protobuf_MessageOptions* msg) { size_t size; google_protobuf_MessageOptions_uninterpreted_option(msg, &size); @@ -6775,6 +7184,23 @@ UPB_INLINE int32_t const* google_protobuf_FieldOptions_targets(const google_prot return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FieldOptions_targets_upb_array(const google_protobuf_FieldOptions* msg, size_t* size) { + const upb_MiniTableField field = {19, UPB_SIZE(28, 32), 0, 4, 14, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FieldOptions_targets_mutable_upb_array(const google_protobuf_FieldOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {19, UPB_SIZE(28, 32), 0, 4, 14, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FieldOptions_has_targets(const google_protobuf_FieldOptions* msg) { size_t size; google_protobuf_FieldOptions_targets(msg, &size); @@ -6795,6 +7221,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Fie return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FieldOptions_uninterpreted_option_upb_array(const google_protobuf_FieldOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(32, 40), 0, 5, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FieldOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_FieldOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(32, 40), 0, 5, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FieldOptions_has_uninterpreted_option(const google_protobuf_FieldOptions* msg) { size_t size; google_protobuf_FieldOptions_uninterpreted_option(msg, &size); @@ -6942,6 +7385,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_One return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_OneofOptions_uninterpreted_option_upb_array(const google_protobuf_OneofOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_OneofOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_OneofOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_OneofOptions_has_uninterpreted_option(const google_protobuf_OneofOptions* msg) { size_t size; google_protobuf_OneofOptions_uninterpreted_option(msg, &size); @@ -7070,6 +7530,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Enu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumOptions_uninterpreted_option_upb_array(const google_protobuf_EnumOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_EnumOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumOptions_has_uninterpreted_option(const google_protobuf_EnumOptions* msg) { size_t size; google_protobuf_EnumOptions_uninterpreted_option(msg, &size); @@ -7180,6 +7657,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Enu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumValueOptions_uninterpreted_option_upb_array(const google_protobuf_EnumValueOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumValueOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_EnumValueOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumValueOptions_has_uninterpreted_option(const google_protobuf_EnumValueOptions* msg) { size_t size; google_protobuf_EnumValueOptions_uninterpreted_option(msg, &size); @@ -7282,6 +7776,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Ser return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ServiceOptions_uninterpreted_option_upb_array(const google_protobuf_ServiceOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ServiceOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_ServiceOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ServiceOptions_has_uninterpreted_option(const google_protobuf_ServiceOptions* msg) { size_t size; google_protobuf_ServiceOptions_uninterpreted_option(msg, &size); @@ -7399,6 +7910,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Met return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_MethodOptions_uninterpreted_option_upb_array(const google_protobuf_MethodOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 8, 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_MethodOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_MethodOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 8, 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_MethodOptions_has_uninterpreted_option(const google_protobuf_MethodOptions* msg) { size_t size; google_protobuf_MethodOptions_uninterpreted_option(msg, &size); @@ -7490,6 +8018,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption_NamePart* const* google_pro return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_UninterpretedOption_name_upb_array(const google_protobuf_UninterpretedOption* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_UninterpretedOption_name_mutable_upb_array(const google_protobuf_UninterpretedOption* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_UninterpretedOption_has_name(const google_protobuf_UninterpretedOption* msg) { size_t size; google_protobuf_UninterpretedOption_name(msg, &size); @@ -7762,6 +8307,23 @@ UPB_INLINE const google_protobuf_SourceCodeInfo_Location* const* google_protobuf return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_location_upb_array(const google_protobuf_SourceCodeInfo* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_location_mutable_upb_array(const google_protobuf_SourceCodeInfo* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_has_location(const google_protobuf_SourceCodeInfo* msg) { size_t size; google_protobuf_SourceCodeInfo_location(msg, &size); @@ -7845,6 +8407,23 @@ UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_path(const goo return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_path_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 8), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_path_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 8), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_path(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_path(msg, &size); @@ -7865,6 +8444,23 @@ UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_span(const goo return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_span_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(8, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_span_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(8, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_span(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_span(msg, &size); @@ -7915,6 +8511,23 @@ UPB_INLINE upb_StringView const* google_protobuf_SourceCodeInfo_Location_leading return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_leading_detached_comments_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(12, 56), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_leading_detached_comments_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(12, 56), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_leading_detached_comments(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_leading_detached_comments(msg, &size); @@ -8052,6 +8665,23 @@ UPB_INLINE const google_protobuf_GeneratedCodeInfo_Annotation* const* google_pro return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_GeneratedCodeInfo_annotation_upb_array(const google_protobuf_GeneratedCodeInfo* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_GeneratedCodeInfo_annotation_mutable_upb_array(const google_protobuf_GeneratedCodeInfo* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_GeneratedCodeInfo_has_annotation(const google_protobuf_GeneratedCodeInfo* msg) { size_t size; google_protobuf_GeneratedCodeInfo_annotation(msg, &size); @@ -8135,6 +8765,23 @@ UPB_INLINE int32_t const* google_protobuf_GeneratedCodeInfo_Annotation_path(cons return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_GeneratedCodeInfo_Annotation_path_upb_array(const google_protobuf_GeneratedCodeInfo_Annotation* msg, size_t* size) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_GeneratedCodeInfo_Annotation_path_mutable_upb_array(const google_protobuf_GeneratedCodeInfo_Annotation* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_path(const google_protobuf_GeneratedCodeInfo_Annotation* msg) { size_t size; google_protobuf_GeneratedCodeInfo_Annotation_path(msg, &size); diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index ebeb0902b42b..a334197865be 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -151,7 +151,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "8c10056074a2698a134dae73dea86ba3d7e0d56b", - sha256 = "c530b25f1afa08016228b7c07066d9cd545b852684bd06dd9235b47d00185b33", + commit = "b399576ccdb97ed37afc045327852b298e32eff3", + sha256 = "fe337a7c2b5fadebfb8e76b4cb3571c9c374efe17aab85e49f3ae71650c3ae98", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 6284b7a41d1c..09b506457b20 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -6841,7 +6841,7 @@ bool upb_MiniTable_Link(upb_MiniTable* mt, const upb_MiniTable** sub_tables, upb_MiniTableField* f = (upb_MiniTableField*)&mt->fields[i]; if (upb_MiniTableField_CType(f) == kUpb_CType_Enum) { const upb_MiniTableEnum* sub = sub_enums[enum_count++]; - if (enum_count > sub_table_count) return false; + if (enum_count > sub_enum_count) return false; if (sub != NULL) { if (!upb_MiniTable_SetSubEnum(mt, f, sub)) return false; } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 8c820ecfc809..7a74b4065b18 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -367,7 +367,7 @@ typedef enum { kUpb_CType_Float = 2, kUpb_CType_Int32 = 3, kUpb_CType_UInt32 = 4, - kUpb_CType_Enum = 5, // Enum values are int32. + kUpb_CType_Enum = 5, // Enum values are int32. TODO(b/279178239): rename kUpb_CType_Message = 6, kUpb_CType_Double = 7, kUpb_CType_Int64 = 8, @@ -2722,7 +2722,8 @@ UPB_API_INLINE int32_t upb_Message_GetInt32(const upb_Message* msg, UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, const upb_MiniTableField* field, int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32); + UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || + upb_MiniTableField_CType(field) == kUpb_CType_Enum); UPB_ASSUME(_upb_MiniTableField_GetRep(field) == kUpb_FieldRep_4Byte); UPB_ASSUME(!upb_IsRepeatedOrMap(field)); return _upb_Message_SetField(msg, field, &value, a); @@ -2748,7 +2749,7 @@ UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, return _upb_Message_SetField(msg, field, &value, a); } -UPB_API_INLINE void upb_Message_SetEnumProto2( +UPB_API_INLINE void upb_Message_SetClosedEnum( upb_Message* msg, const upb_MiniTable* msg_mini_table, const upb_MiniTableField* field, int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(field)); @@ -3474,6 +3475,23 @@ UPB_INLINE const google_protobuf_FileDescriptorProto* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorSet_file_upb_array(const google_protobuf_FileDescriptorSet* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorSet_file_mutable_upb_array(const google_protobuf_FileDescriptorSet* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorSet_has_file(const google_protobuf_FileDescriptorSet* msg) { size_t size; google_protobuf_FileDescriptorSet_file(msg, &size); @@ -3587,6 +3605,23 @@ UPB_INLINE upb_StringView const* google_protobuf_FileDescriptorProto_dependency( return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {3, UPB_SIZE(4, 40), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {3, UPB_SIZE(4, 40), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_dependency(msg, &size); @@ -3607,6 +3642,23 @@ UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_FileDes return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_message_type_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(8, 48), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_message_type_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(8, 48), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_message_type(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_message_type(msg, &size); @@ -3627,6 +3679,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_enum_type_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(12, 56), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_enum_type_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(12, 56), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_enum_type(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_enum_type(msg, &size); @@ -3647,6 +3716,23 @@ UPB_INLINE const google_protobuf_ServiceDescriptorProto* const* google_protobuf_ return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_service_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(16, 64), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_service_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(16, 64), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_service(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_service(msg, &size); @@ -3667,6 +3753,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_Fi return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_extension_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {7, UPB_SIZE(20, 72), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_extension_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {7, UPB_SIZE(20, 72), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_extension(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_extension(msg, &size); @@ -3717,6 +3820,23 @@ UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_public_dependency( return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_public_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {10, UPB_SIZE(32, 96), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_public_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {10, UPB_SIZE(32, 96), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_public_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_public_dependency(msg, &size); @@ -3737,6 +3857,23 @@ UPB_INLINE int32_t const* google_protobuf_FileDescriptorProto_weak_dependency(co return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileDescriptorProto_weak_dependency_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {11, UPB_SIZE(36, 104), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileDescriptorProto_weak_dependency_mutable_upb_array(const google_protobuf_FileDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {11, UPB_SIZE(36, 104), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileDescriptorProto_has_weak_dependency(const google_protobuf_FileDescriptorProto* msg) { size_t size; google_protobuf_FileDescriptorProto_weak_dependency(msg, &size); @@ -4055,6 +4192,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_field_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_field_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_field(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_field(msg, &size); @@ -4075,6 +4229,23 @@ UPB_INLINE const google_protobuf_DescriptorProto* const* google_protobuf_Descrip return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_nested_type_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {3, UPB_SIZE(8, 32), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_nested_type_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {3, UPB_SIZE(8, 32), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_nested_type(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_nested_type(msg, &size); @@ -4095,6 +4266,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto* const* google_protobuf_Des return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_enum_type_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_enum_type_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_enum_type(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_enum_type(msg, &size); @@ -4115,6 +4303,23 @@ UPB_INLINE const google_protobuf_DescriptorProto_ExtensionRange* const* google_p return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_extension_range_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_extension_range_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, 3, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_extension_range(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_extension_range(msg, &size); @@ -4135,6 +4340,23 @@ UPB_INLINE const google_protobuf_FieldDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_extension_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(20, 56), 0, 4, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_extension_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(20, 56), 0, 4, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_extension(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_extension(msg, &size); @@ -4170,6 +4392,23 @@ UPB_INLINE const google_protobuf_OneofDescriptorProto* const* google_protobuf_De return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_oneof_decl_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {8, UPB_SIZE(28, 72), 0, 6, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_oneof_decl_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {8, UPB_SIZE(28, 72), 0, 6, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_oneof_decl(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_oneof_decl(msg, &size); @@ -4190,6 +4429,23 @@ UPB_INLINE const google_protobuf_DescriptorProto_ReservedRange* const* google_pr return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_reserved_range_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {9, UPB_SIZE(32, 80), 0, 7, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_reserved_range_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {9, UPB_SIZE(32, 80), 0, 7, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_reserved_range(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_reserved_range(msg, &size); @@ -4210,6 +4466,23 @@ UPB_INLINE upb_StringView const* google_protobuf_DescriptorProto_reserved_name(c return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_DescriptorProto_reserved_name_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {10, UPB_SIZE(36, 88), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_DescriptorProto_reserved_name_mutable_upb_array(const google_protobuf_DescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {10, UPB_SIZE(36, 88), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_DescriptorProto_has_reserved_name(const google_protobuf_DescriptorProto* msg) { size_t size; google_protobuf_DescriptorProto_reserved_name(msg, &size); @@ -4666,6 +4939,23 @@ UPB_INLINE const google_protobuf_ExtensionRangeOptions_Declaration* const* googl return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ExtensionRangeOptions_declaration_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ExtensionRangeOptions_declaration_mutable_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ExtensionRangeOptions_has_declaration(const google_protobuf_ExtensionRangeOptions* msg) { size_t size; google_protobuf_ExtensionRangeOptions_declaration(msg, &size); @@ -4701,6 +4991,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Ext return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ExtensionRangeOptions_uninterpreted_option_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(12, 16), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ExtensionRangeOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_ExtensionRangeOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(12, 16), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ExtensionRangeOptions_has_uninterpreted_option(const google_protobuf_ExtensionRangeOptions* msg) { size_t size; google_protobuf_ExtensionRangeOptions_uninterpreted_option(msg, &size); @@ -5317,6 +5624,23 @@ UPB_INLINE const google_protobuf_EnumValueDescriptorProto* const* google_protobu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_value_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_value_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_value(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_value(msg, &size); @@ -5352,6 +5676,23 @@ UPB_INLINE const google_protobuf_EnumDescriptorProto_EnumReservedRange* const* g return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_reserved_range_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_reserved_range_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {4, UPB_SIZE(12, 40), 0, 2, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_reserved_range(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_reserved_range(msg, &size); @@ -5372,6 +5713,23 @@ UPB_INLINE upb_StringView const* google_protobuf_EnumDescriptorProto_reserved_na return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumDescriptorProto_reserved_name_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumDescriptorProto_reserved_name_mutable_upb_array(const google_protobuf_EnumDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {5, UPB_SIZE(16, 48), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumDescriptorProto_has_reserved_name(const google_protobuf_EnumDescriptorProto* msg) { size_t size; google_protobuf_EnumDescriptorProto_reserved_name(msg, &size); @@ -5713,6 +6071,23 @@ UPB_INLINE const google_protobuf_MethodDescriptorProto* const* google_protobuf_S return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ServiceDescriptorProto_method_upb_array(const google_protobuf_ServiceDescriptorProto* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ServiceDescriptorProto_method_mutable_upb_array(const google_protobuf_ServiceDescriptorProto* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 24), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ServiceDescriptorProto_has_method(const google_protobuf_ServiceDescriptorProto* msg) { size_t size; google_protobuf_ServiceDescriptorProto_method(msg, &size); @@ -6286,6 +6661,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Fil return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FileOptions_uninterpreted_option_upb_array(const google_protobuf_FileOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(20, 184), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FileOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_FileOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(20, 184), 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FileOptions_has_uninterpreted_option(const google_protobuf_FileOptions* msg) { size_t size; google_protobuf_FileOptions_uninterpreted_option(msg, &size); @@ -6524,6 +6916,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Mes return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_MessageOptions_uninterpreted_option_upb_array(const google_protobuf_MessageOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 8, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_MessageOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_MessageOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 8, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_MessageOptions_has_uninterpreted_option(const google_protobuf_MessageOptions* msg) { size_t size; google_protobuf_MessageOptions_uninterpreted_option(msg, &size); @@ -6777,6 +7186,23 @@ UPB_INLINE int32_t const* google_protobuf_FieldOptions_targets(const google_prot return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FieldOptions_targets_upb_array(const google_protobuf_FieldOptions* msg, size_t* size) { + const upb_MiniTableField field = {19, UPB_SIZE(28, 32), 0, 4, 14, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FieldOptions_targets_mutable_upb_array(const google_protobuf_FieldOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {19, UPB_SIZE(28, 32), 0, 4, 14, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FieldOptions_has_targets(const google_protobuf_FieldOptions* msg) { size_t size; google_protobuf_FieldOptions_targets(msg, &size); @@ -6797,6 +7223,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Fie return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_FieldOptions_uninterpreted_option_upb_array(const google_protobuf_FieldOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(32, 40), 0, 5, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_FieldOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_FieldOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(32, 40), 0, 5, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_FieldOptions_has_uninterpreted_option(const google_protobuf_FieldOptions* msg) { size_t size; google_protobuf_FieldOptions_uninterpreted_option(msg, &size); @@ -6944,6 +7387,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_One return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_OneofOptions_uninterpreted_option_upb_array(const google_protobuf_OneofOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_OneofOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_OneofOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_OneofOptions_has_uninterpreted_option(const google_protobuf_OneofOptions* msg) { size_t size; google_protobuf_OneofOptions_uninterpreted_option(msg, &size); @@ -7072,6 +7532,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Enu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumOptions_uninterpreted_option_upb_array(const google_protobuf_EnumOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_EnumOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumOptions_has_uninterpreted_option(const google_protobuf_EnumOptions* msg) { size_t size; google_protobuf_EnumOptions_uninterpreted_option(msg, &size); @@ -7182,6 +7659,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Enu return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_EnumValueOptions_uninterpreted_option_upb_array(const google_protobuf_EnumValueOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_EnumValueOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_EnumValueOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_EnumValueOptions_has_uninterpreted_option(const google_protobuf_EnumValueOptions* msg) { size_t size; google_protobuf_EnumValueOptions_uninterpreted_option(msg, &size); @@ -7284,6 +7778,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Ser return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_ServiceOptions_uninterpreted_option_upb_array(const google_protobuf_ServiceOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_ServiceOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_ServiceOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_ServiceOptions_has_uninterpreted_option(const google_protobuf_ServiceOptions* msg) { size_t size; google_protobuf_ServiceOptions_uninterpreted_option(msg, &size); @@ -7401,6 +7912,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_Met return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_MethodOptions_uninterpreted_option_upb_array(const google_protobuf_MethodOptions* msg, size_t* size) { + const upb_MiniTableField field = {999, 8, 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_MethodOptions_uninterpreted_option_mutable_upb_array(const google_protobuf_MethodOptions* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {999, 8, 0, 1, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_MethodOptions_has_uninterpreted_option(const google_protobuf_MethodOptions* msg) { size_t size; google_protobuf_MethodOptions_uninterpreted_option(msg, &size); @@ -7492,6 +8020,23 @@ UPB_INLINE const google_protobuf_UninterpretedOption_NamePart* const* google_pro return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_UninterpretedOption_name_upb_array(const google_protobuf_UninterpretedOption* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_UninterpretedOption_name_mutable_upb_array(const google_protobuf_UninterpretedOption* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_UninterpretedOption_has_name(const google_protobuf_UninterpretedOption* msg) { size_t size; google_protobuf_UninterpretedOption_name(msg, &size); @@ -7764,6 +8309,23 @@ UPB_INLINE const google_protobuf_SourceCodeInfo_Location* const* google_protobuf return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_location_upb_array(const google_protobuf_SourceCodeInfo* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_location_mutable_upb_array(const google_protobuf_SourceCodeInfo* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_has_location(const google_protobuf_SourceCodeInfo* msg) { size_t size; google_protobuf_SourceCodeInfo_location(msg, &size); @@ -7847,6 +8409,23 @@ UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_path(const goo return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_path_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 8), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_path_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 8), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_path(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_path(msg, &size); @@ -7867,6 +8446,23 @@ UPB_INLINE int32_t const* google_protobuf_SourceCodeInfo_Location_span(const goo return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_span_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {2, UPB_SIZE(8, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_span_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {2, UPB_SIZE(8, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_span(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_span(msg, &size); @@ -7917,6 +8513,23 @@ UPB_INLINE upb_StringView const* google_protobuf_SourceCodeInfo_Location_leading return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_SourceCodeInfo_Location_leading_detached_comments_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size) { + const upb_MiniTableField field = {6, UPB_SIZE(12, 56), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_SourceCodeInfo_Location_leading_detached_comments_mutable_upb_array(const google_protobuf_SourceCodeInfo_Location* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {6, UPB_SIZE(12, 56), 0, kUpb_NoSub, 12, kUpb_FieldMode_Array | kUpb_LabelFlags_IsAlternate | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_SourceCodeInfo_Location_has_leading_detached_comments(const google_protobuf_SourceCodeInfo_Location* msg) { size_t size; google_protobuf_SourceCodeInfo_Location_leading_detached_comments(msg, &size); @@ -8054,6 +8667,23 @@ UPB_INLINE const google_protobuf_GeneratedCodeInfo_Annotation* const* google_pro return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_GeneratedCodeInfo_annotation_upb_array(const google_protobuf_GeneratedCodeInfo* msg, size_t* size) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_GeneratedCodeInfo_annotation_mutable_upb_array(const google_protobuf_GeneratedCodeInfo* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, 0, 0, 0, 11, kUpb_FieldMode_Array | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_GeneratedCodeInfo_has_annotation(const google_protobuf_GeneratedCodeInfo* msg) { size_t size; google_protobuf_GeneratedCodeInfo_annotation(msg, &size); @@ -8137,6 +8767,23 @@ UPB_INLINE int32_t const* google_protobuf_GeneratedCodeInfo_Annotation_path(cons return NULL; } } +UPB_INLINE const upb_Array* _google_protobuf_GeneratedCodeInfo_Annotation_path_upb_array(const google_protobuf_GeneratedCodeInfo_Annotation* msg, size_t* size) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + const upb_Array* arr = upb_Message_GetArray(msg, &field); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} +UPB_INLINE upb_Array* _google_protobuf_GeneratedCodeInfo_Annotation_path_mutable_upb_array(const google_protobuf_GeneratedCodeInfo_Annotation* msg, size_t* size, upb_Arena* arena) { + const upb_MiniTableField field = {1, UPB_SIZE(4, 16), 0, kUpb_NoSub, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte) << kUpb_FieldRep_Shift)}; + upb_Array* arr = upb_Message_GetOrCreateMutableArray( + (upb_Message*)msg, &field, arena); + if (size) { + *size = arr ? arr->size : 0; + } + return arr; +} UPB_INLINE bool google_protobuf_GeneratedCodeInfo_Annotation_has_path(const google_protobuf_GeneratedCodeInfo_Annotation* msg) { size_t size; google_protobuf_GeneratedCodeInfo_Annotation_path(msg, &size); From b21d6be78bb2586c7882c67ba975cc3a3e785862 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 25 Apr 2023 13:47:12 -0700 Subject: [PATCH 426/463] Updating version.json and repo version numbers to: 23.0-rc1 --- CMakeLists.txt | 2 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/README.md | 6 ++--- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 2 +- java/util/pom.xml | 2 +- php/ext/google/protobuf/protobuf.h | 2 +- protobuf_version.bzl | 10 ++++---- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 ++-- src/google/protobuf/any.pb.h | 4 ++-- src/google/protobuf/api.pb.h | 4 ++-- src/google/protobuf/compiler/plugin.pb.h | 4 ++-- src/google/protobuf/descriptor.pb.h | 4 ++-- src/google/protobuf/duration.pb.h | 4 ++-- src/google/protobuf/empty.pb.h | 4 ++-- src/google/protobuf/field_mask.pb.h | 4 ++-- src/google/protobuf/port_def.inc | 10 ++++---- src/google/protobuf/source_context.pb.h | 4 ++-- src/google/protobuf/struct.pb.h | 4 ++-- src/google/protobuf/stubs/common.h | 10 ++++---- src/google/protobuf/timestamp.pb.h | 4 ++-- src/google/protobuf/type.pb.h | 4 ++-- src/google/protobuf/wrappers.pb.h | 4 ++-- version.json | 24 +++++++++---------- 35 files changed, 73 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22a7feda9a3a..266b01b322b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,7 +93,7 @@ if (protobuf_BUILD_SHARED_LIBS) endif () # Version metadata -set(protobuf_VERSION_STRING "4.22.3") +set(protobuf_VERSION_STRING "4.23.0-rc-1") set(protobuf_DESCRIPTION "Protocol Buffers") set(protobuf_CONTACT "protobuf@googlegroups.com") diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index b3f6c2d440df..249dad51f1a4 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '4.22.3' + s.version = '4.23.0-rc1' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index ea5606ddfe1b..6f35be69f281 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.22.3' + s.version = '3.23.0-rc1' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 129bff4fe0ba..2655b87eb8f4 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.22.3 + 3.23.0-rc1 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index c9dfd8fe471c..b1fc5717f5c9 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -5,7 +5,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.22.3 + 3.23.0-rc1 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/java/README.md b/java/README.md index 91bf8a2e23a3..32ad5850c473 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.22.3 + 3.23.0-rc-1 ``` @@ -37,14 +37,14 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.22.3 + 3.23.0-rc-1 ``` ### Gradle If you are using Gradle, add the following to your `build.gradle` file's -dependencies: `implementation 'com.google.protobuf:protobuf-java:3.22.3'` Again, +dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-1'` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 6656dcfc936e..9e0190766dc2 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.22.3 + 3.23.0-RC1 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 38f8be575cf8..80afa2e77787 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 protobuf-java diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index d9b34d412d2a..a0ad298cdeca 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index c5c25be41ae5..09849a8f33f9 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index 94ab4df2d837..7d392e44eebf 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.22.3 + 3.23.0-rc-1 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 36bc12a8f2b5..76db105dabf4 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 2143f823510d..fd83571fceee 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index dae6ffd5fd9d..ca46caa29640 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.22.3 + 3.23.0-RC1 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index e5f42a8ca6b3..aeb307e9d76a 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.22.3 + 3.23.0-RC1 protobuf-java-util diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index 4a941f45def5..cde54a30e168 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.22.3" +#define PHP_PROTOBUF_VERSION "3.23.0RC1" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 74e6f992e258..68548e5d4c77 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,6 +1,6 @@ """ Contains version numbers to be used in other bzl files """ -PROTOC_VERSION = "22.3" -PROTOBUF_JAVA_VERSION = "3.22.3" -PROTOBUF_PYTHON_VERSION = "4.22.3" -PROTOBUF_PHP_VERSION = "3.22.3" -PROTOBUF_RUBY_VERSION = "3.22.3" +PROTOC_VERSION = "23.0-rc-1" +PROTOBUF_JAVA_VERSION = "3.23.0-RC1" +PROTOBUF_PYTHON_VERSION = "4.23.0rc1" +PROTOBUF_PHP_VERSION = "3.23.0RC1" +PROTOBUF_RUBY_VERSION = "3.23.0.rc.1" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 10e07e248b4b..1af0fe8ab08f 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.22.3' +__version__ = '4.23.0rc1' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 65b105d49cf1..49fcd6fa9404 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.22.3" + s.version = "3.23.0.rc.1" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index 1cc8eb5bb878..c2d3263bf8a9 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.22.3 + 3.23.0-RC1 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.22.3 + 3.23.0-RC1 org.jruby diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 7cfd9a3db05e..168c5ea7d8fc 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index d9e9addba946..70396247fc8e 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 1595de536e76..e9bf2a47966a 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index bcb766e47142..80fd85b99fa4 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 9dc53e789572..39ae3c5a4c27 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 9316dfb0b73c..59564a7ddff0 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 4c8ecbf5a8c8..81a8c317d4b5 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 0862c9533e6f..17b1b5e68dce 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -231,22 +231,22 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 4022003 +#define PROTOBUF_VERSION 4023000 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined #endif -#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3021000 +#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 4023000 #ifdef PROTOBUF_MIN_PROTOC_VERSION #error PROTOBUF_MIN_PROTOC_VERSION was previously defined #endif -#define PROTOBUF_MIN_PROTOC_VERSION 3021000 +#define PROTOBUF_MIN_PROTOC_VERSION 4023000 #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined #endif -#define PROTOBUF_VERSION_SUFFIX "" +#define PROTOBUF_VERSION_SUFFIX "-rc1" #if defined(PROTOBUF_NAMESPACE) || defined(PROTOBUF_NAMESPACE_ID) #error PROTOBUF_NAMESPACE or PROTOBUF_NAMESPACE_ID was previously defined @@ -434,7 +434,7 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // The minimum library version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3021000 +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 4023000 #ifdef PROTOBUF_RTTI #error PROTOBUF_RTTI was previously defined diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index c28575639535..82e56f440212 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index ff46f9968e83..b907aebfabf6 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index ec8dd4428e1d..a35696a3b033 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -66,23 +66,23 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 4022003 +#define GOOGLE_PROTOBUF_VERSION 4023000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. -#define GOOGLE_PROTOBUF_VERSION_SUFFIX "" +#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc1" // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 3021000; +static const int kMinHeaderVersionForLibrary = 4023000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3021000 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 4023000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 3021000; +static const int kMinHeaderVersionForProtoc = 4023000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index cd6cb62f3e09..f134d6068564 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 224f9ce59a8b..dfce43e444fd 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index e4d75441d4c7..bafb1b056e43 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -9,13 +9,13 @@ #include #include "google/protobuf/port_def.inc" -#if PROTOBUF_VERSION < 3021000 +#if PROTOBUF_VERSION < 4023000 #error "This file was generated by a newer version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please update" #error "your headers." #endif // PROTOBUF_VERSION -#if 4022003 < PROTOBUF_MIN_PROTOC_VERSION +#if 4023000 < PROTOBUF_MIN_PROTOC_VERSION #error "This file was generated by an older version of protoc which is" #error "incompatible with your Protocol Buffer headers. Please" #error "regenerate this file with a newer version of protoc." diff --git a/version.json b/version.json index adac99583e7a..19aaeca9a80c 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { - "main": { - "protoc_version": "23-dev", + "23.x": { + "protoc_version": "23.0-rc1", "lts": false, - "date": "2023-01-26", + "date": "2023-04-25", "languages": { - "cpp": "4.23-dev", - "csharp": "3.23-dev", - "java": "3.23-dev", - "javascript": "3.23-dev", - "objectivec": "3.23-dev", - "php": "3.23-dev", - "python": "4.23-dev", - "ruby": "3.23-dev" + "cpp": "4.23.0-rc1", + "csharp": "3.23.0-rc1", + "java": "3.23.0-rc1", + "javascript": "3.23.0-rc1", + "objectivec": "3.23.0-rc1", + "php": "3.23.0-rc1", + "python": "4.23.0-rc1", + "ruby": "3.23.0-rc1" } } -} +} \ No newline at end of file From 04a1a52892d74e053546e9b07d56c4815bb27ea1 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 25 Apr 2023 13:47:12 -0700 Subject: [PATCH 427/463] Updating version.json to: 23.0-dev --- version.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/version.json b/version.json index 19aaeca9a80c..77301baf906b 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-rc1", + "protoc_version": "23.0-dev", "lts": false, "date": "2023-04-25", "languages": { - "cpp": "4.23.0-rc1", - "csharp": "3.23.0-rc1", - "java": "3.23.0-rc1", - "javascript": "3.23.0-rc1", - "objectivec": "3.23.0-rc1", - "php": "3.23.0-rc1", - "python": "4.23.0-rc1", - "ruby": "3.23.0-rc1" + "cpp": "4.23.0-dev", + "csharp": "3.23.0-dev", + "java": "3.23.0-dev", + "javascript": "3.23.0-dev", + "objectivec": "3.23.0-dev", + "php": "3.23.0-dev", + "python": "4.23.0-dev", + "ruby": "3.23.0-dev" } } } \ No newline at end of file From 2230d0e7482e3a51e96b6f83fa2e63b0ea66c1ba Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 26 Apr 2023 17:24:38 -0700 Subject: [PATCH 428/463] Fix homebrew issue on Mac runners causing PHP CI failures PiperOrigin-RevId: 527419345 --- .github/workflows/test_php.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index d6f352c605f7..45bdc5136cfe 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -154,14 +154,17 @@ jobs: - name: Update submodules run: git submodule update --init --remote --recursive + - name: Uninstall problematic libgd + run: brew uninstall --ignore-dependencies gd + - name: Install dependencies - run: brew install coreutils # For sha256sum + run: brew install coreutils gd - name: Pin PHP version uses: shivammathur/setup-php@d30ad8b1843ace22e6698ab99bbafaa747b6bd0d # 2.24.0 with: php-version: ${{ matrix.version }} - + - name: Check PHP version run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1) From db1d3f8b5160d7f70cc8deeb565d040b0834412d Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 26 Apr 2023 17:35:57 -0700 Subject: [PATCH 429/463] Replace previous breaking changes with deprecation warnings. These changes hid public APIs that are deprecated in advance of protobuf editions. They will be fully removed in a future release. Any uses of them should be migrated to the future-facing feature helpers we provide on descriptor classes. PiperOrigin-RevId: 527422059 --- .../java/com/google/protobuf/Descriptors.java | 4 ++ src/google/protobuf/descriptor.cc | 57 +++++++++++-------- src/google/protobuf/descriptor.h | 45 ++++++++++++++- src/google/protobuf/descriptor_legacy.h | 13 ++++- 4 files changed, 92 insertions(+), 27 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index be7c76a5b52c..e866d199b3ca 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -162,6 +162,8 @@ public List getPublicDependencies() { } /** The syntax of the .proto file. */ + @Deprecated + public enum Syntax { UNKNOWN("unknown"), PROTO2("proto2"), @@ -175,6 +177,8 @@ enum Syntax { } /** Get the syntax of the .proto file. */ + @Deprecated + public Syntax getSyntax() { if (Syntax.PROTO3.name.equals(proto.getSyntax())) { return Syntax.PROTO3; diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index c12e9dcbce0b..51b6dd9a01c8 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -2612,10 +2612,12 @@ void FileDescriptor::CopyHeadingTo(FileDescriptorProto* proto) const { } // TODO(liujisi): Also populate when syntax="proto2". - if (syntax() == SYNTAX_PROTO3 + FileDescriptorLegacy::Syntax syntax = FileDescriptorLegacy(this).syntax(); + if (syntax == FileDescriptorLegacy::Syntax::SYNTAX_PROTO3 ) { - proto->set_syntax(SyntaxName(syntax())); + proto->set_syntax(FileDescriptorLegacy::SyntaxName(syntax)); } + if (&options() != &FileOptions::default_instance()) { *proto->mutable_options() = options(); } @@ -3010,8 +3012,9 @@ std::string FileDescriptor::DebugStringWithOptions( SourceLocationCommentPrinter syntax_comment(this, path, "", debug_string_options); syntax_comment.AddPreComment(&contents); - absl::SubstituteAndAppend(&contents, "syntax = \"$0\";\n\n", - SyntaxName(syntax())); + absl::SubstituteAndAppend( + &contents, "syntax = \"$0\";\n\n", + FileDescriptorLegacy::SyntaxName(FileDescriptorLegacy(this).syntax())); syntax_comment.AddPostComment(&contents); } @@ -3282,7 +3285,7 @@ void FieldDescriptor::DebugString( // Label is omitted for maps, oneof, and plain proto3 fields. if (is_map() || real_containing_oneof() || - (is_optional() && !has_optional_keyword())) { + (is_optional() && !FieldDescriptorLegacy(this).has_optional_keyword())) { label.clear(); } @@ -3573,7 +3576,8 @@ bool FileDescriptor::GetSourceLocation(SourceLocation* out_location) const { bool FieldDescriptor::is_packed() const { if (!is_packable()) return false; - if (file_->syntax() == FileDescriptor::SYNTAX_PROTO2) { + if (FileDescriptorLegacy(file_).syntax() == + FileDescriptorLegacy::Syntax::SYNTAX_PROTO2) { return (options_ != nullptr) && options_->packed(); } else { return options_ == nullptr || !options_->has_packed() || options_->packed(); @@ -3582,7 +3586,8 @@ bool FieldDescriptor::is_packed() const { bool FieldDescriptor::requires_utf8_validation() const { return type() == TYPE_STRING && - file()->syntax() == FileDescriptor::SYNTAX_PROTO3; + FileDescriptorLegacy(file_).syntax() == + FileDescriptorLegacy::Syntax::SYNTAX_PROTO3; } bool Descriptor::GetSourceLocation(SourceLocation* out_location) const { @@ -3989,7 +3994,7 @@ class DescriptorBuilder { const Descriptor* result); void CheckFieldJsonNameUniqueness(const std::string& message_name, const DescriptorProto& message, - FileDescriptor::Syntax syntax, + FileDescriptorLegacy::Syntax syntax, bool use_custom_names); void CheckEnumValueUniqueness(const EnumDescriptorProto& proto, const EnumDescriptor* result); @@ -4711,7 +4716,7 @@ FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( placeholder->tables_ = &FileDescriptorTables::GetEmptyInstance(); placeholder->source_code_info_ = &SourceCodeInfo::default_instance(); placeholder->is_placeholder_ = true; - placeholder->syntax_ = FileDescriptor::SYNTAX_UNKNOWN; + placeholder->syntax_ = FileDescriptorLegacy::SYNTAX_UNKNOWN; placeholder->finished_building_ = true; // All other fields are zero or nullptr. @@ -5226,11 +5231,11 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // TODO(liujisi): Report error when the syntax is empty after all the protos // have added the syntax statement. if (proto.syntax().empty() || proto.syntax() == "proto2") { - file_->syntax_ = FileDescriptor::SYNTAX_PROTO2; + file_->syntax_ = FileDescriptorLegacy::SYNTAX_PROTO2; } else if (proto.syntax() == "proto3") { - file_->syntax_ = FileDescriptor::SYNTAX_PROTO3; + file_->syntax_ = FileDescriptorLegacy::SYNTAX_PROTO3; } else { - file_->syntax_ = FileDescriptor::SYNTAX_UNKNOWN; + file_->syntax_ = FileDescriptorLegacy::SYNTAX_UNKNOWN; AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, [&] { return absl::StrCat("Unrecognized syntax: ", proto.syntax()); }); @@ -5681,11 +5686,12 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, void DescriptorBuilder::CheckFieldJsonNameUniqueness( const DescriptorProto& proto, const Descriptor* result) { - FileDescriptor::Syntax syntax = result->file()->syntax(); + FileDescriptorLegacy::Syntax syntax = + FileDescriptorLegacy(result->file()).syntax(); std::string message_name = result->full_name(); if (pool_->deprecated_legacy_json_field_conflicts_ || IsLegacyJsonFieldConflictEnabled(result->options())) { - if (syntax == FileDescriptor::SYNTAX_PROTO3) { + if (syntax == FileDescriptorLegacy::Syntax::SYNTAX_PROTO3) { // Only check default JSON names for conflicts in proto3. This is legacy // behavior that will be removed in a later version. CheckFieldJsonNameUniqueness(message_name, proto, syntax, false); @@ -5725,7 +5731,7 @@ bool JsonNameLooksLikeExtension(std::string name) { void DescriptorBuilder::CheckFieldJsonNameUniqueness( const std::string& message_name, const DescriptorProto& message, - FileDescriptor::Syntax syntax, bool use_custom_names) { + FileDescriptorLegacy::Syntax syntax, bool use_custom_names) { absl::flat_hash_map name_to_field; for (const FieldDescriptorProto& field : message.field()) { JsonNameDetails details = GetJsonNameDetails(&field, use_custom_names); @@ -5769,7 +5775,7 @@ void DescriptorBuilder::CheckFieldJsonNameUniqueness( }; bool involves_default = !details.is_custom || !match.is_custom; - if (syntax == FileDescriptor::SYNTAX_PROTO2 && involves_default) { + if (syntax == FileDescriptorLegacy::SYNTAX_PROTO2 && involves_default) { // TODO(b/261750676) Upgrade this to an error once downstream protos have // been fixed. AddWarning(message_name, field, DescriptorPool::ErrorCollector::NAME, @@ -5808,7 +5814,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, result->proto3_optional_ = proto.proto3_optional(); if (proto.proto3_optional() && - file_->syntax() != FileDescriptor::SYNTAX_PROTO3) { + FileDescriptorLegacy(file_).syntax() != + FileDescriptorLegacy::Syntax::SYNTAX_PROTO3) { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, [&] { return absl::StrCat( @@ -6221,7 +6228,8 @@ void DescriptorBuilder::CheckEnumValueUniqueness( // compatibility we issue only a warning for proto2. if ((pool_->deprecated_legacy_json_field_conflicts_ || IsLegacyJsonFieldConflictEnabled(result->options())) && - result->file()->syntax() == FileDescriptor::SYNTAX_PROTO2) { + FileDescriptorLegacy(result->file()).syntax() == + FileDescriptorLegacy::Syntax::SYNTAX_PROTO2) { AddWarning(value->full_name(), proto.value(i), DescriptorPool::ErrorCollector::NAME, make_error); continue; @@ -6585,7 +6593,7 @@ void DescriptorBuilder::CrossLinkMessage(Descriptor* message, const FieldDescriptor* field = message->field(i); if (field->proto3_optional_) { if (!field->containing_oneof() || - !field->containing_oneof()->is_synthetic()) { + !OneofDescriptorLegacy(field->containing_oneof()).is_synthetic()) { AddError(message->full_name(), proto.field(i), DescriptorPool::ErrorCollector::OTHER, "Fields with proto3_optional set must be " @@ -6598,7 +6606,7 @@ void DescriptorBuilder::CrossLinkMessage(Descriptor* message, int first_synthetic = -1; for (int i = 0; i < message->oneof_decl_count(); i++) { const OneofDescriptor* oneof = message->oneof_decl(i); - if (oneof->is_synthetic()) { + if (OneofDescriptorLegacy(oneof).is_synthetic()) { if (first_synthetic == -1) { first_synthetic = i; } @@ -7140,7 +7148,8 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file, } } } - if (file->syntax() == FileDescriptor::SYNTAX_PROTO3) { + if (FileDescriptorLegacy(file).syntax() == + FileDescriptorLegacy::Syntax::SYNTAX_PROTO3) { ValidateProto3(file, proto); } } @@ -7203,8 +7212,10 @@ void DescriptorBuilder::ValidateProto3Field(FieldDescriptor* field, } if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM && field->enum_type() && - field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3 && - field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_UNKNOWN) { + FileDescriptorLegacy(field->enum_type()->file()).syntax() != + FileDescriptorLegacy::Syntax::SYNTAX_PROTO3 && + FileDescriptorLegacy(field->enum_type()->file()).syntax() != + FileDescriptorLegacy::Syntax::SYNTAX_UNKNOWN) { // Proto3 messages can only use Proto3 enum types; otherwise we can't // guarantee that the default value is zero. AddError( diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 97829d2e2f1c..347f51abfe02 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -79,6 +79,8 @@ #ifdef SWIG #define PROTOBUF_EXPORT +#define PROTOBUF_IGNORE_DEPRECATION_START +#define PROTOBUF_IGNORE_DEPRECATION_STOP #endif @@ -922,10 +924,17 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase { friend class Reflection; friend class FieldDescriptorLegacy; + + public: + ABSL_DEPRECATED( + "Syntax is deprecated in favor of editions, please use " + "FieldDescriptor::has_presence instead.") // Returns true if this field was syntactically written with "optional" in the // .proto file. Excludes singular proto3 fields that do not have a label. bool has_optional_keyword() const; + private: + // Fill the json_name field of FieldDescriptorProto. void CopyJsonNameTo(FieldDescriptorProto* proto) const; @@ -1083,10 +1092,17 @@ class PROTOBUF_EXPORT OneofDescriptor : private internal::SymbolBase { friend class compiler::cpp::Formatter; friend class OneofDescriptorLegacy; + + public: + ABSL_DEPRECATED( + "Syntax is deprecated in favor of editions, please use " + "real_oneof_decl_count for now instead of is_synthetic.") // Returns whether this oneof was inserted by the compiler to wrap a proto3 // optional field. If this returns true, code generators should *not* emit it. bool is_synthetic() const; + private: + // See Descriptor::DebugString(). void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; @@ -1657,12 +1673,18 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase { // descriptor.proto, and any available extensions of that message. const FileOptions& options() const; + private: // With the upcoming release of editions, syntax should not be used for // business logic. Instead, the various feature helpers defined in this file // should be used to query more targeted behaviors. For example: // has_presence, is_closed, requires_utf8_validation. - enum Syntax + enum + ABSL_DEPRECATED( + "Syntax is deprecated in favor of editions. Please use targeted " + "feature helpers instead (e.g. has_presence, is_packed, " + "requires_utf8_validation, etc).") + Syntax #ifndef SWIG : int #endif // !SWIG @@ -1671,13 +1693,22 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase { SYNTAX_PROTO2 = 2, SYNTAX_PROTO3 = 3, }; + PROTOBUF_IGNORE_DEPRECATION_START + ABSL_DEPRECATED( + "Syntax is deprecated in favor of editions. Please use targeted " + "feature helpers instead (e.g. has_presence, is_packed, " + "requires_utf8_validation, etc).") Syntax syntax() const; + PROTOBUF_IGNORE_DEPRECATION_STOP // Define a visibility-restricted wrapper for internal use until the migration // is complete. friend class FileDescriptorLegacy; + PROTOBUF_IGNORE_DEPRECATION_START + ABSL_DEPRECATED("Syntax is deprecated in favor of editions") static const char* SyntaxName(Syntax syntax); + PROTOBUF_IGNORE_DEPRECATION_STOP public: @@ -2314,7 +2345,9 @@ PROTOBUF_DEFINE_ARRAY_ACCESSOR(EnumDescriptor, reserved_range, PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, reserved_name_count, int) inline bool EnumDescriptor::is_closed() const { + PROTOBUF_IGNORE_DEPRECATION_START return file()->syntax() != FileDescriptor::SYNTAX_PROTO3; + PROTOBUF_IGNORE_DEPRECATION_STOP } PROTOBUF_DEFINE_NAME_ACCESSOR(EnumValueDescriptor) @@ -2465,25 +2498,33 @@ inline bool FieldDescriptor::is_map() const { } inline bool FieldDescriptor::has_optional_keyword() const { + PROTOBUF_IGNORE_DEPRECATION_START return proto3_optional_ || (file()->syntax() == FileDescriptor::SYNTAX_PROTO2 && is_optional() && !containing_oneof()); + PROTOBUF_IGNORE_DEPRECATION_STOP } inline const OneofDescriptor* FieldDescriptor::real_containing_oneof() const { + PROTOBUF_IGNORE_DEPRECATION_START auto* oneof = containing_oneof(); return oneof && !oneof->is_synthetic() ? oneof : nullptr; + PROTOBUF_IGNORE_DEPRECATION_STOP } inline bool FieldDescriptor::has_presence() const { + PROTOBUF_IGNORE_DEPRECATION_START if (is_repeated()) return false; return cpp_type() == CPPTYPE_MESSAGE || containing_oneof() || file()->syntax() == FileDescriptor::SYNTAX_PROTO2; + PROTOBUF_IGNORE_DEPRECATION_STOP } inline bool FieldDescriptor::legacy_enum_field_treated_as_closed() const { + PROTOBUF_IGNORE_DEPRECATION_START return type() == TYPE_ENUM && file()->syntax() == FileDescriptor::SYNTAX_PROTO2; + PROTOBUF_IGNORE_DEPRECATION_STOP } // To save space, index() is computed by looking at the descriptor's position @@ -2586,9 +2627,11 @@ inline const FileDescriptor* FileDescriptor::weak_dependency(int index) const { return dependency(weak_dependencies_[index]); } +PROTOBUF_IGNORE_DEPRECATION_START inline FileDescriptor::Syntax FileDescriptor::syntax() const { return static_cast(syntax_); } +PROTOBUF_IGNORE_DEPRECATION_STOP namespace internal { diff --git a/src/google/protobuf/descriptor_legacy.h b/src/google/protobuf/descriptor_legacy.h index 763f4477e8fd..75b9cfa9ba25 100644 --- a/src/google/protobuf/descriptor_legacy.h +++ b/src/google/protobuf/descriptor_legacy.h @@ -66,6 +66,7 @@ namespace google { namespace protobuf { +PROTOBUF_IGNORE_DEPRECATION_START // Wraps FileDescriptor. class PROTOBUF_EXPORT FileDescriptorLegacy { @@ -74,10 +75,15 @@ class PROTOBUF_EXPORT FileDescriptorLegacy { // Any dependencies on file-level syntax keyword should be replaced by // feature-level switches to support go/protobuf-editions. - using Syntax = FileDescriptor::Syntax; - Syntax syntax() const { return desc_->syntax(); } + enum Syntax { + SYNTAX_UNKNOWN = FileDescriptor::SYNTAX_UNKNOWN, + SYNTAX_PROTO2 = FileDescriptor::SYNTAX_PROTO2, + SYNTAX_PROTO3 = FileDescriptor::SYNTAX_PROTO3, + }; + Syntax syntax() const { return static_cast(desc_->syntax()); } static absl::string_view SyntaxName(Syntax syntax) { - return FileDescriptor::SyntaxName(syntax); + return FileDescriptor::SyntaxName( + static_cast(syntax)); } private: @@ -104,6 +110,7 @@ class PROTOBUF_EXPORT OneofDescriptorLegacy { const OneofDescriptor* desc_; }; +PROTOBUF_IGNORE_DEPRECATION_STOP } // namespace protobuf } // namespace google From 112a9021bef147387dbf6ea0cac9b506debafdd6 Mon Sep 17 00:00:00 2001 From: Mike Kruskal <62662355+mkruskal-google@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:28:42 -0700 Subject: [PATCH 430/463] Squashed 'third_party/utf8_range/' changes from 72c943dea..cd1507d14 (#12583) cd1507d14 Fix typo in pkg-config file to properly link the utf8_range libraries 3f179d134 Add pkg-config file declaring Abseil dependency of utf8_range git-subtree-dir: third_party/utf8_range git-subtree-split: cd1507d1479815fbcd8ff24dc05a978a62098bae --- third_party/utf8_range/cmake/utf8_range.pc.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/third_party/utf8_range/cmake/utf8_range.pc.cmake b/third_party/utf8_range/cmake/utf8_range.pc.cmake index 5801b43e82f1..64b4b770b297 100644 --- a/third_party/utf8_range/cmake/utf8_range.pc.cmake +++ b/third_party/utf8_range/cmake/utf8_range.pc.cmake @@ -7,6 +7,5 @@ Name: UTF8 Range Description: Google's UTF8 Library Version: 1.0 Requires: absl_strings -Libs: -L${libdir} -lprotobuf-lite @CMAKE_THREAD_LIBS_INIT@ +Libs: -L${libdir} -lutf8_validity -lutf8_range @CMAKE_THREAD_LIBS_INIT@ Cflags: -I${includedir} -Conflicts: protobuf From 433c35dbcb70248f1627d83ad7c88487e600bb55 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 27 Apr 2023 11:54:50 -0700 Subject: [PATCH 431/463] Remove osgi dependency from main java libraries. //java:core, //java:lite, and //java:util shouldn't have this extra dependency. For now, we just add a separate version but we will look into using the java_library in protobuf_versioned_java_library in the future. PiperOrigin-RevId: 527643325 --- java/core/BUILD.bazel | 37 +++++++++++++++++++++++++++++++++---- java/util/BUILD.bazel | 20 ++++++++++++++++++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 49bf0e0af9eb..8f2bfe346296 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -121,11 +121,21 @@ internal_gen_well_known_protos_java( ) # Should be used as `//java/lite`. -protobuf_versioned_java_library( +java_library( name = "lite", srcs = LITE_SRCS + [ ":gen_well_known_protos_javalite", ], + visibility = [ + "//java/lite:__pkg__", + ], +) + +protobuf_versioned_java_library( + name = "lite_bundle", + srcs = LITE_SRCS + [ + ":gen_well_known_protos_javalite", + ], bundle_description = "Lite version of Protocol Buffers library. This " + "version is optimized for code size, but does not " + "guarantee API/ABI stability.", @@ -145,7 +155,7 @@ protobuf_java_export( "//:lite_well_known_protos", ], tags = ["manual"], - runtime_deps = [":lite"], + runtime_deps = [":lite_bundle"], ) protobuf_java_library( @@ -171,7 +181,7 @@ internal_gen_well_known_protos_java( ], ) -protobuf_versioned_java_library( +java_library( name = "core", srcs = glob( [ @@ -181,6 +191,25 @@ protobuf_versioned_java_library( ) + [ ":gen_well_known_protos_java", ], + visibility = ["//visibility:public"], + exports = [ + ":lite_runtime_only", + ], + deps = [ + ":lite_runtime_only", + ], +) + +protobuf_versioned_java_library( + name = "core_bundle", + srcs = glob( + [ + "src/main/java/com/google/protobuf/*.java", + ], + exclude = LITE_SRCS, + ) + [ + ":gen_well_known_protos_java", + ], bundle_description = "Core Protocol Buffers library. Protocol Buffers " + "are a way of encoding structured data in an " + "efficient yet extensible format.", @@ -205,7 +234,7 @@ protobuf_java_export( "//src/google/protobuf:descriptor_proto_srcs", ], tags = ["manual"], - runtime_deps = [":core"], + runtime_deps = [":core_bundle"], ) filegroup( diff --git a/java/util/BUILD.bazel b/java/util/BUILD.bazel index 1bb23587c2a5..c6ab799a5d50 100644 --- a/java/util/BUILD.bazel +++ b/java/util/BUILD.bazel @@ -5,11 +5,27 @@ load("//build_defs:java_opts.bzl", "protobuf_java_export", "protobuf_versioned_j load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") load("//java/internal:testing.bzl", "junit_tests") -protobuf_versioned_java_library( +java_library( name = "util", srcs = glob([ "src/main/java/com/google/protobuf/util/*.java", ]), + visibility = ["//visibility:public"], + deps = [ + "//java/core", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_code_gson_gson", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + "@maven//:com_google_j2objc_j2objc_annotations", + ], +) + +protobuf_versioned_java_library( + name = "util_bundle", + srcs = glob([ + "src/main/java/com/google/protobuf/util/*.java", + ]), bundle_description = "Utilities for Protocol Buffers", bundle_name = "Protocol Buffers [Util]", bundle_symbolic_name = "com.google.protobuf.util", @@ -32,7 +48,7 @@ protobuf_java_export( pom_template = "pom_template.xml", tags = ["manual"], visibility = ["//java:__pkg__"], - runtime_deps = [":util"], + runtime_deps = [":util_bundle"], ) filegroup( From e67e58a8e9bf9826608a2ae3c92bda027cab7911 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 27 Apr 2023 09:29:52 -0700 Subject: [PATCH 432/463] Silence deprecation warnings on FileDescriptor::SyntaxName PiperOrigin-RevId: 527600194 --- src/google/protobuf/descriptor.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 51b6dd9a01c8..0d77da558c5b 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -825,6 +825,7 @@ const char* const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { "repeated", // LABEL_REPEATED }; +PROTOBUF_IGNORE_DEPRECATION_START const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { switch (syntax) { case SYNTAX_PROTO2: @@ -837,6 +838,7 @@ const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { ABSL_LOG(FATAL) << "can't reach here."; return nullptr; } +PROTOBUF_IGNORE_DEPRECATION_STOP static const char* const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty"; From 99c9491f7cf586ec470b24b66a77c32b449c8e48 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 27 Apr 2023 23:21:50 +0000 Subject: [PATCH 433/463] Fix maven pom test --- .github/workflows/test_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index da1680c9ad7e..1ebbcbd7bb1d 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -112,7 +112,7 @@ jobs: shell: bash run: | LOCAL_MAVEN_GROUP_DIR="${HOME}/.m2/repository/com/google/protobuf" - VERSION=$(grep 'Bundle-Version:' ./java/core/target/classes/META-INF/MANIFEST.MF |awk '{print $2}' |tr -d '\r') + VERSION=$(grep "" bazel-bin/java/core/core_mvn-pom.xml | sed "s/\(.*\)<\/version>/\1/" | xargs) cp core_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java/${VERSION}/protobuf-java-${VERSION}.pom cp util_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java-util/${VERSION}/protobuf-java-util-${VERSION}.pom - name: Clean up From d19c7997ab658a0a69bb1b61c73c3f36d2d21259 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 27 Apr 2023 23:32:47 +0000 Subject: [PATCH 434/463] Fix bom tests --- .github/workflows/test_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 1ebbcbd7bb1d..d8a8522263c1 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -112,7 +112,7 @@ jobs: shell: bash run: | LOCAL_MAVEN_GROUP_DIR="${HOME}/.m2/repository/com/google/protobuf" - VERSION=$(grep "" bazel-bin/java/core/core_mvn-pom.xml | sed "s/\(.*\)<\/version>/\1/" | xargs) + VERSION=$(grep "" core_mvn-pom.xml | sed "s/\(.*\)<\/version>/\1/" | xargs) cp core_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java/${VERSION}/protobuf-java-${VERSION}.pom cp util_mvn-pom.xml ${LOCAL_MAVEN_GROUP_DIR}/protobuf-java-util/${VERSION}/protobuf-java-util-${VERSION}.pom - name: Clean up From 49bb3f20647b914fc52909eec19f260fb9a945f3 Mon Sep 17 00:00:00 2001 From: Mike Kruskal <62662355+mkruskal-google@users.noreply.github.com> Date: Fri, 28 Apr 2023 08:57:36 -0700 Subject: [PATCH 435/463] Fixes Clang 6 linker bug (#12600) PiperOrigin-RevId: 527711840 --- src/google/protobuf/map_field.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index c48e7646d2c3..70b12b1e7965 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -604,7 +604,7 @@ class MapField final : public TypeDefinedMapFieldBase { constexpr MapField() : impl_() {} MapField(const MapField&) = delete; MapField& operator=(const MapField&) = delete; - ~MapField() = default; + ~MapField() {} explicit MapField(Arena* arena) : TypeDefinedMapFieldBase(arena), impl_(arena) {} From 0b73dd44a3257c788f930bbb068ffdcb98679456 Mon Sep 17 00:00:00 2001 From: Mike Kruskal <62662355+mkruskal-google@users.noreply.github.com> Date: Fri, 28 Apr 2023 08:57:46 -0700 Subject: [PATCH 436/463] Add -ex switch to all docker bash commands. (#12601) Without this, certain failures will be silently ignored. PiperOrigin-RevId: 527718574 --- .github/workflows/test_cpp.yml | 2 +- .github/workflows/test_csharp.yml | 4 ++-- .github/workflows/test_php.yml | 2 +- .github/workflows/test_ruby.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index c5b42be3a5d5..4a8a8f72e7f5 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -192,7 +192,7 @@ jobs: platform: linux/386 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -c ' + /bin/bash -cex ' cd /workspace; ccache -z; cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}; diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index cc729ed0d5fd..62580ca97eed 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -62,7 +62,7 @@ jobs: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -c ' + /bin/bash -cex ' DOTNET_CLI_TELEMETRY_OPTOUT=true DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true dotnet publish -c Release -f net6.0 /workspace/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj' @@ -80,7 +80,7 @@ jobs: skip-staleness-check: true credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -c ' + /bin/bash -cex ' DOTNET_CLI_TELEMETRY_OPTOUT=true DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true dotnet vstest /workspace/csharp/src/Google.Protobuf.Test/bin/Release/net6.0/publish/Google.Protobuf.Test.dll' diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 45bdc5136cfe..f94e3e7514ff 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -95,7 +95,7 @@ jobs: platform: linux/386 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -c ' + /bin/bash -cex ' cd php && php -v && php -m; composer update --ignore-platform-reqs; PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 33906f5e72e3..45b9a1e0cfb1 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -63,7 +63,7 @@ jobs: skip-staleness-check: true credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -c ' + /bin/bash -cex ' gem install bundler; cd /workspace/ruby; bundle; From eb96482b3ab926ca5493310f66abb3badd7b49ec Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Fri, 28 Apr 2023 16:44:01 +0000 Subject: [PATCH 437/463] Update UPB dep --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index a334197865be..4f30e0000935 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -151,7 +151,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "b399576ccdb97ed37afc045327852b298e32eff3", - sha256 = "fe337a7c2b5fadebfb8e76b4cb3571c9c374efe17aab85e49f3ae71650c3ae98", + commit = "c50109929e015b5c180787385212aa32ca9a0daa", + sha256 = "e083c3bd7886c67ea1633444f4a387f02a082e2b5dd93ec8d98e37653634fcbf", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) From 51f51ac4efc3a1146e10416d31d24e0052cd9d86 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 28 Apr 2023 10:25:19 -0700 Subject: [PATCH 438/463] Updating version.json and repo version numbers to: 23.0-rc2 --- CMakeLists.txt | 2 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/README.md | 6 +++--- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 2 +- java/util/pom.xml | 2 +- php/ext/google/protobuf/protobuf.h | 2 +- protobuf_version.bzl | 10 +++++----- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 ++-- src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/stubs/common.h | 2 +- version.json | 20 +++++++++---------- 23 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 266b01b322b2..1b628a818d8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,7 +93,7 @@ if (protobuf_BUILD_SHARED_LIBS) endif () # Version metadata -set(protobuf_VERSION_STRING "4.23.0-rc-1") +set(protobuf_VERSION_STRING "4.23.0-rc-2") set(protobuf_DESCRIPTION "Protocol Buffers") set(protobuf_CONTACT "protobuf@googlegroups.com") diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 249dad51f1a4..baafd5f3ecac 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '4.23.0-rc1' + s.version = '4.23.0-rc2' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index 6f35be69f281..6bd42607d0b5 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.23.0-rc1' + s.version = '3.23.0-rc2' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 2655b87eb8f4..163ff3121323 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.23.0-rc1 + 3.23.0-rc2 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index b1fc5717f5c9..e908b6f29683 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -5,7 +5,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.23.0-rc1 + 3.23.0-rc2 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/java/README.md b/java/README.md index 32ad5850c473..fd761dc7e85d 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.23.0-rc-1 + 3.23.0-rc-2 ``` @@ -37,14 +37,14 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.23.0-rc-1 + 3.23.0-rc-2 ``` ### Gradle If you are using Gradle, add the following to your `build.gradle` file's -dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-1'` Again, +dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-2'` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 9e0190766dc2..9c4c533604bd 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.23.0-RC1 + 3.23.0-RC2 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 80afa2e77787..adace564543f 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 protobuf-java diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index a0ad298cdeca..85509cb5967c 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 09849a8f33f9..2562e12e336a 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index 7d392e44eebf..1d677b5b4c92 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.23.0-rc-1 + 3.23.0-rc-2 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 76db105dabf4..10e98f550c33 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index fd83571fceee..0194e661a17c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index ca46caa29640..34165fe67f58 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.23.0-RC1 + 3.23.0-RC2 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index aeb307e9d76a..7d9b6c0735ea 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC1 + 3.23.0-RC2 protobuf-java-util diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index cde54a30e168..f5c4c70d3a63 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.23.0RC1" +#define PHP_PROTOBUF_VERSION "3.23.0RC2" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 68548e5d4c77..2f9a9f5d968e 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,6 +1,6 @@ """ Contains version numbers to be used in other bzl files """ -PROTOC_VERSION = "23.0-rc-1" -PROTOBUF_JAVA_VERSION = "3.23.0-RC1" -PROTOBUF_PYTHON_VERSION = "4.23.0rc1" -PROTOBUF_PHP_VERSION = "3.23.0RC1" -PROTOBUF_RUBY_VERSION = "3.23.0.rc.1" +PROTOC_VERSION = "23.0-rc-2" +PROTOBUF_JAVA_VERSION = "3.23.0-RC2" +PROTOBUF_PYTHON_VERSION = "4.23.0rc2" +PROTOBUF_PHP_VERSION = "3.23.0RC2" +PROTOBUF_RUBY_VERSION = "3.23.0.rc.2" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 1af0fe8ab08f..47fc9b3421a4 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.23.0rc1' +__version__ = '4.23.0rc2' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 49fcd6fa9404..937ab3e25d3f 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.23.0.rc.1" + s.version = "3.23.0.rc.2" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index c2d3263bf8a9..0025923264cc 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.23.0-RC1 + 3.23.0-RC2 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.23.0-RC1 + 3.23.0-RC2 org.jruby diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 17b1b5e68dce..12f0b4991dfa 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -246,7 +246,7 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined #endif -#define PROTOBUF_VERSION_SUFFIX "-rc1" +#define PROTOBUF_VERSION_SUFFIX "-rc2" #if defined(PROTOBUF_NAMESPACE) || defined(PROTOBUF_NAMESPACE_ID) #error PROTOBUF_NAMESPACE or PROTOBUF_NAMESPACE_ID was previously defined diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index a35696a3b033..67d476ed3ecc 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -69,7 +69,7 @@ namespace internal { #define GOOGLE_PROTOBUF_VERSION 4023000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. -#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc1" +#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc2" // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code diff --git a/version.json b/version.json index 77301baf906b..4e996cec68b1 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-dev", + "protoc_version": "23.0-rc2", "lts": false, - "date": "2023-04-25", + "date": "2023-04-28", "languages": { - "cpp": "4.23.0-dev", - "csharp": "3.23.0-dev", - "java": "3.23.0-dev", - "javascript": "3.23.0-dev", - "objectivec": "3.23.0-dev", - "php": "3.23.0-dev", - "python": "4.23.0-dev", - "ruby": "3.23.0-dev" + "cpp": "4.23.0-rc2", + "csharp": "3.23.0-rc2", + "java": "3.23.0-rc2", + "javascript": "3.23.0-rc2", + "objectivec": "3.23.0-rc2", + "php": "3.23.0-rc2", + "python": "4.23.0-rc2", + "ruby": "3.23.0-rc2" } } } \ No newline at end of file From 29b8dc0eef0fc6daaf7948dc4820855966f45945 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 28 Apr 2023 10:25:19 -0700 Subject: [PATCH 439/463] Updating version.json to: 23.0-dev --- version.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/version.json b/version.json index 4e996cec68b1..4ec814607468 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-rc2", + "protoc_version": "23.0-dev", "lts": false, "date": "2023-04-28", "languages": { - "cpp": "4.23.0-rc2", - "csharp": "3.23.0-rc2", - "java": "3.23.0-rc2", - "javascript": "3.23.0-rc2", - "objectivec": "3.23.0-rc2", - "php": "3.23.0-rc2", - "python": "4.23.0-rc2", - "ruby": "3.23.0-rc2" + "cpp": "4.23.0-dev", + "csharp": "3.23.0-dev", + "java": "3.23.0-dev", + "javascript": "3.23.0-dev", + "objectivec": "3.23.0-dev", + "php": "3.23.0-dev", + "python": "4.23.0-dev", + "ruby": "3.23.0-dev" } } } \ No newline at end of file From 46fb4aa8d2ade5e0067fce271fcb5293c5c70500 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 1 May 2023 14:07:57 +0000 Subject: [PATCH 440/463] fix(libprotoc): export useful symbols from .so --- src/google/protobuf/compiler/cpp/helpers.h | 18 +++++++-------- src/google/protobuf/compiler/cpp/names.h | 27 +++++++++++----------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index 4f09e1fbddee..d9ccb8d81464 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -119,10 +119,10 @@ std::string Namespace(const FileDescriptor* d, const Options& options); std::string Namespace(const Descriptor* d, const Options& options); std::string Namespace(const FieldDescriptor* d, const Options& options); std::string Namespace(const EnumDescriptor* d, const Options& options); -std::string Namespace(const FileDescriptor* d); -std::string Namespace(const Descriptor* d); -std::string Namespace(const FieldDescriptor* d); -std::string Namespace(const EnumDescriptor* d); +PROTOC_EXPORT std::string Namespace(const FileDescriptor* d); +PROTOC_EXPORT std::string Namespace(const Descriptor* d); +PROTOC_EXPORT std::string Namespace(const FieldDescriptor* d); +PROTOC_EXPORT std::string Namespace(const EnumDescriptor* d); class MessageSCCAnalyzer; @@ -136,14 +136,14 @@ bool CanClearByZeroing(const FieldDescriptor* field); bool HasTrivialSwap(const FieldDescriptor* field, const Options& options, MessageSCCAnalyzer* scc_analyzer); -std::string ClassName(const Descriptor* descriptor); -std::string ClassName(const EnumDescriptor* enum_descriptor); +PROTOC_EXPORT std::string ClassName(const Descriptor* descriptor); +PROTOC_EXPORT std::string ClassName(const EnumDescriptor* enum_descriptor); std::string QualifiedClassName(const Descriptor* d, const Options& options); std::string QualifiedClassName(const EnumDescriptor* d, const Options& options); -std::string QualifiedClassName(const Descriptor* d); -std::string QualifiedClassName(const EnumDescriptor* d); +PROTOC_EXPORT std::string QualifiedClassName(const Descriptor* d); +PROTOC_EXPORT std::string QualifiedClassName(const EnumDescriptor* d); // DEPRECATED just use ClassName or QualifiedClassName, a boolean is very // unreadable at the callsite. @@ -215,7 +215,7 @@ std::string ResolveKeyword(absl::string_view name); // The name is coerced to lower-case to emulate proto1 behavior. People // should be using lowercase-with-underscores style for proto field names // anyway, so normally this just returns field->name(). -std::string FieldName(const FieldDescriptor* field); +PROTOC_EXPORT std::string FieldName(const FieldDescriptor* field); // Returns the (unqualified) private member name for this field in C++ code. std::string FieldMemberName(const FieldDescriptor* field, bool split); diff --git a/src/google/protobuf/compiler/cpp/names.h b/src/google/protobuf/compiler/cpp/names.h index 150649377b1e..7afc76943497 100644 --- a/src/google/protobuf/compiler/cpp/names.h +++ b/src/google/protobuf/compiler/cpp/names.h @@ -57,10 +57,10 @@ namespace cpp { // message Baz { message Moo {} } // Then the qualified namespace for Moo would be: // ::foo::bar -std::string Namespace(const FileDescriptor* d); -std::string Namespace(const Descriptor* d); -std::string Namespace(const FieldDescriptor* d); -std::string Namespace(const EnumDescriptor* d); +PROTOC_EXPORT std::string Namespace(const FileDescriptor* d); +PROTOC_EXPORT std::string Namespace(const Descriptor* d); +PROTOC_EXPORT std::string Namespace(const FieldDescriptor* d); +PROTOC_EXPORT std::string Namespace(const EnumDescriptor* d); // Returns the unqualified C++ name. // @@ -69,8 +69,8 @@ std::string Namespace(const EnumDescriptor* d); // message Baz { message Moo {} } // Then the non-qualified version would be: // Baz_Moo -std::string ClassName(const Descriptor* descriptor); -std::string ClassName(const EnumDescriptor* enum_descriptor); +PROTOC_EXPORT std::string ClassName(const Descriptor* descriptor); +PROTOC_EXPORT std::string ClassName(const EnumDescriptor* enum_descriptor); // Returns the fully qualified C++ name. // @@ -79,25 +79,26 @@ std::string ClassName(const EnumDescriptor* enum_descriptor); // message Baz { message Moo {} } // Then the qualified ClassName for Moo would be: // ::foo::bar::Baz_Moo -std::string QualifiedClassName(const Descriptor* d); -std::string QualifiedClassName(const EnumDescriptor* d); -std::string QualifiedExtensionName(const FieldDescriptor* d); +PROTOC_EXPORT std::string QualifiedClassName(const Descriptor* d); +PROTOC_EXPORT std::string QualifiedClassName(const EnumDescriptor* d); +PROTOC_EXPORT std::string QualifiedExtensionName(const FieldDescriptor* d); // Get the (unqualified) name that should be used for this field in C++ code. // The name is coerced to lower-case to emulate proto1 behavior. People // should be using lowercase-with-underscores style for proto field names // anyway, so normally this just returns field->name(). -std::string FieldName(const FieldDescriptor* field); +PROTOC_EXPORT std::string FieldName(const FieldDescriptor* field); // Requires that this field is in a oneof. Returns the (unqualified) case // constant for this field. -std::string OneofCaseConstantName(const FieldDescriptor* field); +PROTOC_EXPORT std::string OneofCaseConstantName(const FieldDescriptor* field); // Returns the quafilied case constant for this field. -std::string QualifiedOneofCaseConstantName(const FieldDescriptor* field); +PROTOC_EXPORT std::string QualifiedOneofCaseConstantName( + const FieldDescriptor* field); // Get the (unqualified) name that should be used for this enum value in C++ // code. -std::string EnumValueName(const EnumValueDescriptor* enum_value); +PROTOC_EXPORT std::string EnumValueName(const EnumValueDescriptor* enum_value); // Strips ".proto" or ".protodevel" from the end of a filename. PROTOC_EXPORT std::string StripProto(absl::string_view filename); From ad2dd26ffc6858b7a0e991d75df55897cc24b2e5 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Tue, 2 May 2023 17:20:46 +0000 Subject: [PATCH 441/463] Mark internal rule manual --- objectivec/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/objectivec/BUILD.bazel b/objectivec/BUILD.bazel index 0965de5b276f..87f1ec04d19d 100644 --- a/objectivec/BUILD.bazel +++ b/objectivec/BUILD.bazel @@ -45,6 +45,7 @@ genrule( for ext in _OBJC_EXTS ]), exec_tools = ["//:protoc"], + tags = ["manual"], ) staleness_test( From b36c39236e43f4ab9c1472064b6161d00aef21c5 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Mon, 1 May 2023 15:11:07 -0700 Subject: [PATCH 442/463] Fix bug in _internal_copy_files where the rule would fail in downstream repositories. Taken from https://github.com/protocolbuffers/upb/blob/main/bazel/protobuf.patch#L42 Fixes https://github.com/protocolbuffers/protobuf/issues/12620. PiperOrigin-RevId: 528586464 --- python/internal.bzl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/python/internal.bzl b/python/internal.bzl index 7441721e9d9d..8f99becec4ce 100644 --- a/python/internal.bzl +++ b/python/internal.bzl @@ -1,4 +1,12 @@ -# Internal helpers for building the Python protobuf runtime. +""" +Internal helpers for building the Python protobuf runtime. +""" + +def _remove_cross_repo_path(path): + components = path.split("/") + if components[0] == "..": + return "/".join(components[2:]) + return path def _internal_copy_files_impl(ctx): strip_prefix = ctx.attr.strip_prefix @@ -7,10 +15,11 @@ def _internal_copy_files_impl(ctx): src_dests = [] for src in ctx.files.srcs: - if src.short_path[:len(strip_prefix)] != strip_prefix: + short_path = _remove_cross_repo_path(src.short_path) + if short_path[:len(strip_prefix)] != strip_prefix: fail("Source does not start with %s: %s" % - (strip_prefix, src.short_path)) - dest = ctx.actions.declare_file(src.short_path[len(strip_prefix):]) + (strip_prefix, short_path)) + dest = ctx.actions.declare_file(short_path[len(strip_prefix):]) src_dests.append([src, dest]) if ctx.attr.is_windows: From b40633ff0bf9b34bf3bec9f3d35eec2d951a98b8 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 2 May 2023 10:08:56 -0700 Subject: [PATCH 443/463] Turn off `clang::musttail` on i386 We found last week that Clang's support for `musttail` on 32-bit Linux does not seem to work reliably, so let's disable it for now. PiperOrigin-RevId: 528817403 --- src/google/protobuf/port_def.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 12f0b4991dfa..a14b89cfcb64 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -322,9 +322,9 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_TAILCALL #error PROTOBUF_TAILCALL was previously defined #endif -#if __has_cpp_attribute(clang::musttail) && !defined(__arm__) && \ - !defined(_ARCH_PPC) && !defined(__wasm__) && \ - !(defined(_MSC_VER) && defined(_M_IX86)) && \ +#if __has_cpp_attribute(clang::musttail) && !defined(__arm__) && \ + !defined(_ARCH_PPC) && !defined(__wasm__) && \ + !(defined(_MSC_VER) && defined(_M_IX86)) && !defined(__i386__) && \ !(defined(__NDK_MAJOR__) && __NDK_MAJOR <= 24) # ifndef PROTO2_OPENSOURCE // Compilation fails on ARM32: b/195943306 From c4731cdfdd2451dd88901a9645e2db0878fdc62e Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Tue, 2 May 2023 11:18:25 -0700 Subject: [PATCH 444/463] Update abseil dependency and reorder dependencies to ensure we use the version specified in protobuf_deps. PiperOrigin-RevId: 528838071 --- WORKSPACE | 8 ++++---- protobuf_deps.bzl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 65796e340db7..414548395b46 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -7,6 +7,10 @@ local_repository( path = "examples", ) +# Load common dependencies first to ensure we use the correct version +load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") +protobuf_deps() + # Bazel platform rules. http_archive( name = "platforms", @@ -28,10 +32,6 @@ load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") googletest_deps() -# Load common dependencies. -load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") -protobuf_deps() - load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") rules_jvm_external_deps() diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 4f30e0000935..3ce3ee485796 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -42,8 +42,8 @@ def protobuf_deps(): _github_archive( name = "com_google_absl", repo = "https://github.com/abseil/abseil-cpp", - commit = "78be63686ba732b25052be15f8d6dee891c05749", # Abseil LTS 20230125 - sha256 = "4f356a07b9ec06ef51f943928508566e992f621ed5fa4dd588865d7bed1284cd", + commit = "b971ac5250ea8de900eae9f95e06548d14cd95fe", # Abseil LTS 20230125.2 + sha256 = "f7c2cb2c5accdcbbbd5c0c59f241a988c0b1da2a3b7134b823c0bd613b1a6880", ) if not native.existing_rule("zlib"): From df32e78d45a466404a1b8424be4b013043d98a8d Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Wed, 3 May 2023 16:13:00 +0000 Subject: [PATCH 445/463] Update UPB dep --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 3ce3ee485796..20c90a07a6db 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -151,7 +151,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "c50109929e015b5c180787385212aa32ca9a0daa", - sha256 = "e083c3bd7886c67ea1633444f4a387f02a082e2b5dd93ec8d98e37653634fcbf", + commit = "7c4a2bcd27f16c93085022efbe40776ed223de7a", + sha256 = "c0ee8ac3fa47b1fc244f8ed4704561a51cc200f394697a493f7f33182ede24b5", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) From 5ded06d9abd36a5a1f38e76a804608f5baf8e1ae Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Wed, 3 May 2023 13:48:06 -0700 Subject: [PATCH 446/463] Automated rollback of commit a2c5ea614cd5330de4b56a9937377f65309a31f8. PiperOrigin-RevId: 529185812 --- objectivec/BUILD.bazel | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/objectivec/BUILD.bazel b/objectivec/BUILD.bazel index 87f1ec04d19d..10798380698c 100644 --- a/objectivec/BUILD.bazel +++ b/objectivec/BUILD.bazel @@ -7,11 +7,9 @@ load(":defs.bzl", "objc_proto_camel_case_name") # The WKTs have to be checked in to support the CocoaPods and Xcode builds. This # generule and test ensure the source are current. # -# Within the ":objectivec" target, the outputs of the genrule are then used to -# ensure they are always "current". This implementation is basically the same -# has how the WKTs are handled in src/google/protobuf/BUILD.bazel for the C++ -# version. They share the potential downsides around layer checks and that -# someone could #include the header with the `wkt/` prefix on the name. +# TODO: Improve the bazel build so it uses these generated headers so it is +# always current, and only the builds that can't easily build protoc and +# generate the files rely on the checked in ones. _WELL_KNOWN_TYPES = [ "any", @@ -62,6 +60,16 @@ staleness_test( objc_library( name = "objectivec", hdrs = [ + "GPBAny.pbobjc.h", + "GPBApi.pbobjc.h", + "GPBDuration.pbobjc.h", + "GPBEmpty.pbobjc.h", + "GPBFieldMask.pbobjc.h", + "GPBSourceContext.pbobjc.h", + "GPBStruct.pbobjc.h", + "GPBTimestamp.pbobjc.h", + "GPBType.pbobjc.h", + "GPBWrappers.pbobjc.h", "GPBArray.h", "GPBBootstrap.h", "GPBCodedInputStream.h", @@ -102,30 +110,39 @@ objc_library( "GPBUnknownFieldSet_PackagePrivate.h", "GPBUnknownField_PackagePrivate.h", "GPBUtilities_PackagePrivate.h", - ] + ["wkt/GPB" + wkt + ".pbobjc.h" for wkt in _OBJC_WKT_NAMES], + ], copts = [ "-Wno-vla", ], includes = [ ".", - "wkt", ], non_arc_srcs = [ + "GPBAny.pbobjc.m", + "GPBApi.pbobjc.m", "GPBArray.m", "GPBCodedInputStream.m", "GPBCodedOutputStream.m", "GPBDescriptor.m", "GPBDictionary.m", + "GPBDuration.pbobjc.m", + "GPBEmpty.pbobjc.m", "GPBExtensionInternals.m", "GPBExtensionRegistry.m", + "GPBFieldMask.pbobjc.m", "GPBMessage.m", "GPBRootObject.m", + "GPBSourceContext.pbobjc.m", + "GPBStruct.pbobjc.m", + "GPBTimestamp.pbobjc.m", + "GPBType.pbobjc.m", "GPBUnknownField.m", "GPBUnknownFieldSet.m", "GPBUtilities.m", "GPBWellKnownTypes.m", "GPBWireFormat.m", - ] + ["wkt/GPB" + wkt + ".pbobjc.m" for wkt in _OBJC_WKT_NAMES], + "GPBWrappers.pbobjc.m", + ], target_compatible_with = select({ "@platforms//os:macos": [], "@platforms//os:ios": [], From 000755a8a7701df3615bd12c90802a8fd26e7809 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 4 May 2023 16:18:48 +0000 Subject: [PATCH 447/463] Update UPB dep --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 20c90a07a6db..a7addef55304 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -151,7 +151,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "7c4a2bcd27f16c93085022efbe40776ed223de7a", - sha256 = "c0ee8ac3fa47b1fc244f8ed4704561a51cc200f394697a493f7f33182ede24b5", + commit = "af735076087f77099835f546572556a91c97b047", + sha256 = "df8d48190198511e0d91a46643684aaba6f0930f74eb9eee40506ceaa9f4ad7d", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) From 645336fb8a37916bd4abf206e38f04f3a4a95c1a Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 4 May 2023 16:58:31 +0000 Subject: [PATCH 448/463] Fix tests --- .github/workflows/test_cpp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 4a8a8f72e7f5..492c5046868c 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -123,7 +123,7 @@ jobs: # Initialize/update the submodule from remote to validate a new fetch based on .gitmodules # We have to do this separately since actions/checkout doesn't support the "--remote" flag - name: Update submodules - run: git submodule update --init --remote --recursive + run: git submodule update --init --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 From 74cea3dda5bfcc310e2facef9df7084951116190 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 4 May 2023 17:35:06 +0000 Subject: [PATCH 449/463] Revert "Fix tests" This reverts commit 645336fb8a37916bd4abf206e38f04f3a4a95c1a. --- .github/workflows/test_cpp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 492c5046868c..4a8a8f72e7f5 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -123,7 +123,7 @@ jobs: # Initialize/update the submodule from remote to validate a new fetch based on .gitmodules # We have to do this separately since actions/checkout doesn't support the "--remote" flag - name: Update submodules - run: git submodule update --init --recursive + run: git submodule update --init --remote --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 From edbee27994ee9349728d03f6eeb1de47edc6f06c Mon Sep 17 00:00:00 2001 From: Mike Kruskal <62662355+mkruskal-google@users.noreply.github.com> Date: Thu, 4 May 2023 10:54:32 -0700 Subject: [PATCH 450/463] Bump Abseil submodule to 20230125.3 (#12660) * Bump Abseil submodule to 20230125.3 * Fix gtest installation setup with submodules --- cmake/gtest.cmake | 6 +++--- third_party/abseil-cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index 5562380cae6a..cf516d6d53b8 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -32,9 +32,9 @@ else() "GTEST_CREATE_SHARED_LIBRARY=1" ) - if (protobuf_INSTALL) - set(protobuf_INSTALL_TESTS ON) - endif() + endif() + if (protobuf_INSTALL) + set(protobuf_INSTALL_TESTS ON) endif() target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT}) diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index 78be63686ba7..c2435f8342c2 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit 78be63686ba732b25052be15f8d6dee891c05749 +Subproject commit c2435f8342c2d0ed8101cb43adfd605fdc52dca2 From 6f3c80b1a0f98865a4c7281bde51ffb34e6f30df Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 4 May 2023 09:22:57 -0700 Subject: [PATCH 451/463] Internal change PiperOrigin-RevId: 529429609 --- cmake/gtest.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index cf516d6d53b8..5562380cae6a 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -32,9 +32,9 @@ else() "GTEST_CREATE_SHARED_LIBRARY=1" ) - endif() - if (protobuf_INSTALL) - set(protobuf_INSTALL_TESTS ON) + if (protobuf_INSTALL) + set(protobuf_INSTALL_TESTS ON) + endif() endif() target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT}) From 750a6e7d7cbd8c022e18834f0a57fcd76d2c3c58 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 4 May 2023 10:57:15 -0700 Subject: [PATCH 452/463] Bump Abseil submodule to 20230125.3 (#12660) This partially addresses https://github.com/protocolbuffers/protobuf/issues/12201 Closes #12660 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12660 from mkruskal-google:absl-update ce07c45a1027b9736f8c111c8aae864ac242108f PiperOrigin-RevId: 529456996 --- cmake/gtest.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index 5562380cae6a..cf516d6d53b8 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -32,9 +32,9 @@ else() "GTEST_CREATE_SHARED_LIBRARY=1" ) - if (protobuf_INSTALL) - set(protobuf_INSTALL_TESTS ON) - endif() + endif() + if (protobuf_INSTALL) + set(protobuf_INSTALL_TESTS ON) endif() target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT}) From 24d3e74ac95737ac8642bfcdef3fa464e7d15d10 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 4 May 2023 11:06:37 -0700 Subject: [PATCH 453/463] Remove --remote argument in cmake tests. This will prevent non-hermetic breakages from changes to the Abseil release branch we're pinned to. PiperOrigin-RevId: 529459765 --- .github/workflows/test_cpp.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 4a8a8f72e7f5..1b4a18c9526f 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -123,7 +123,7 @@ jobs: # Initialize/update the submodule from remote to validate a new fetch based on .gitmodules # We have to do this separately since actions/checkout doesn't support the "--remote" flag - name: Update submodules - run: git submodule update --init --remote --recursive + run: git submodule update --init --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 @@ -147,7 +147,7 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Update submodules - run: git submodule update --init --remote --recursive + run: git submodule update --init --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 @@ -178,7 +178,7 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Update submodules - run: git submodule update --init --remote --recursive + run: git submodule update --init --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 @@ -309,7 +309,7 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Update submodules - run: git submodule update --init --remote --recursive + run: git submodule update --init --recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 From 5f97a7f494c1294619d4b34db4a13c0f443f03b7 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 4 May 2023 12:57:03 -0700 Subject: [PATCH 454/463] Define a new set of cmake tests. These more closely follow the standard practices of our users, where dependencies are pre-installed instead of using our provided sub-modules. This will prevent issues such as #12201 from reoccuring. Additionally, this cl bumps our Abseil dependency to the latest release, and fixes a GTest issue that went previously unnoticed. PiperOrigin-RevId: 529490402 --- .github/workflows/test_cpp.yml | 117 +++++++++++++++++---------------- cmake/gtest.cmake | 2 +- protobuf_deps.bzl | 4 +- 3 files changed, 63 insertions(+), 60 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 1b4a18c9526f..70f3a75a91dd 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -94,23 +94,11 @@ jobs: fail-fast: false # Don't cancel all jobs if one fails. matrix: include: - - command: >- - /test.sh - -Dprotobuf_BUILD_CONFORMANCE=ON - -Dprotobuf_BUILD_EXAMPLES=ON - -DCMAKE_CXX_STANDARD=14 + - flags: -Dprotobuf_BUILD_EXAMPLES=ON - name: Ninja - command: >- - /test.sh - -G Ninja - -Dprotobuf_BUILD_CONFORMANCE=ON - -DCMAKE_CXX_STANDARD=14 + flags: -G Ninja - name: Shared - command: >- - /test.sh - -Dprotobuf_BUILD_CONFORMANCE=ON - -Dprotobuf_BUILD_SHARED_LIBS=ON - -DCMAKE_CXX_STANDARD=14 + flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON name: Linux CMake ${{ matrix.name}} runs-on: ubuntu-latest @@ -120,11 +108,6 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - # Initialize/update the submodule from remote to validate a new fetch based on .gitmodules - # We have to do this separately since actions/checkout doesn't support the "--remote" flag - - name: Update submodules - run: git submodule update --init --recursive - - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 with: @@ -133,9 +116,13 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: ${{ matrix.command }} ${{ env.CCACHE_CMAKE_FLAGS }} + command: >- + /test.sh ${{ matrix.flags}} ${{ env.CCACHE_CMAKE_FLAGS }} + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=ON + -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package + linux-cmake-install: name: Linux CMake Install @@ -145,9 +132,7 @@ jobs: uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: ref: ${{ inputs.safe-checkout }} - - - name: Update submodules - run: git submodule update --init --recursive + submodules: recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 @@ -157,19 +142,20 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} \&\& + /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package \&\& /test.sh ${{ env.CCACHE_CMAKE_FLAGS }} -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 + -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package - linux-cmake-32-bit: - name: Linux CMake 32-bit + linux-cmake-examples: + name: Linux CMake Examples runs-on: ubuntu-latest steps: - name: Checkout pending changes @@ -177,31 +163,27 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - - name: Update submodules - run: git submodule update --init --recursive - - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 with: - cache-prefix: linux-cmake-32-bit + cache-prefix: linux-cmake-examples - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea - platform: linux/386 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /bin/bash -cex ' - cd /workspace; - ccache -z; - cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}; - cmake --build . --parallel 20; - ctest --verbose --parallel 20; - ccache -s' + /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} + -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package + -Dprotobuf_BUILD_EXAMPLES=OFF \&\& + cd examples \&\& + mkdir build \&\& + cmake -S . -B build -DCMAKE_CXX_STANDARD=14 \&\& + cmake --build build - linux-cmake-examples: - name: Linux CMake Examples + linux-cmake-submodules: + name: Linux CMake Submodules runs-on: ubuntu-latest steps: - name: Checkout pending changes @@ -213,7 +195,7 @@ jobs: - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 with: - cache-prefix: linux-cmake-examples + cache-prefix: linux-cmake-${{ matrix.name }} - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 @@ -221,15 +203,38 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} \&\& - cd examples \&\& - mkdir build \&\& - cmake - ${{ env.CCACHE_CMAKE_FLAGS }} - -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF - -DCMAKE_CXX_STANDARD=14 - -S . -B build \&\& - cmake --build build + /test.sh ${{ env.CCACHE_CMAKE_FLAGS }} + -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 + + linux-cmake-32-bit: + name: Linux CMake 32-bit + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + submodules: recursive + + - name: Setup ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 + with: + cache-prefix: linux-cmake-32-bit + + - name: Run tests + uses: protocolbuffers/protobuf-ci/docker@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea + platform: linux/386 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /bin/bash -cex ' + cd /workspace; + ccache -z; + cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}; + cmake --build . --parallel 20; + ctest --verbose --parallel 20; + ccache -s' non-linux: strategy: @@ -307,9 +312,7 @@ jobs: uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: ref: ${{ inputs.safe-checkout }} - - - name: Update submodules - run: git submodule update --init --recursive + submodules: recursive - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index cf516d6d53b8..b891db9fe3c7 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -1,7 +1,7 @@ option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one in third_party/googletest)" OFF) if (protobuf_USE_EXTERNAL_GTEST) - find_package(GTest REQUIRED) + find_package(GTest REQUIRED CONFIG) else() if (NOT EXISTS "${protobuf_SOURCE_DIR}/third_party/googletest/CMakeLists.txt") message(FATAL_ERROR diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 20c90a07a6db..338ce513fe24 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -42,8 +42,8 @@ def protobuf_deps(): _github_archive( name = "com_google_absl", repo = "https://github.com/abseil/abseil-cpp", - commit = "b971ac5250ea8de900eae9f95e06548d14cd95fe", # Abseil LTS 20230125.2 - sha256 = "f7c2cb2c5accdcbbbd5c0c59f241a988c0b1da2a3b7134b823c0bd613b1a6880", + commit = "c2435f8342c2d0ed8101cb43adfd605fdc52dca2", # Abseil LTS 20230125.3 + sha256 = "ea1d31db00eb37e607bfda17ffac09064670ddf05da067944c4766f517876390", ) if not native.existing_rule("zlib"): From d8f44cc2fb69e3d74943b922b0a5d34ab4b9cc31 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 5 May 2023 09:17:00 -0700 Subject: [PATCH 455/463] Updating version.json and repo version numbers to: 23.0-rc3 --- CMakeLists.txt | 2 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/README.md | 6 +++--- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 2 +- java/util/pom.xml | 2 +- php/ext/google/protobuf/protobuf.h | 2 +- protobuf_version.bzl | 10 +++++----- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 ++-- src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/stubs/common.h | 2 +- version.json | 20 +++++++++---------- 23 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b628a818d8f..3f2f0ba063e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,7 +93,7 @@ if (protobuf_BUILD_SHARED_LIBS) endif () # Version metadata -set(protobuf_VERSION_STRING "4.23.0-rc-2") +set(protobuf_VERSION_STRING "4.23.0-rc-3") set(protobuf_DESCRIPTION "Protocol Buffers") set(protobuf_CONTACT "protobuf@googlegroups.com") diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index baafd5f3ecac..a76d480c67df 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '4.23.0-rc2' + s.version = '4.23.0-rc3' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index 6bd42607d0b5..bbed94f833f4 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.23.0-rc2' + s.version = '3.23.0-rc3' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 163ff3121323..99d8adf235ab 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.23.0-rc2 + 3.23.0-rc3 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index e908b6f29683..6fb30ba2a119 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -5,7 +5,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.23.0-rc2 + 3.23.0-rc3 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/java/README.md b/java/README.md index fd761dc7e85d..fb7ed24a23fc 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.23.0-rc-2 + 3.23.0-rc-3 ``` @@ -37,14 +37,14 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.23.0-rc-2 + 3.23.0-rc-3 ``` ### Gradle If you are using Gradle, add the following to your `build.gradle` file's -dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-2'` Again, +dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-3'` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 9c4c533604bd..569b63651e64 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.23.0-RC2 + 3.23.0-RC3 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index adace564543f..8f5ece64a614 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 protobuf-java diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index 85509cb5967c..482ea7bfbd8f 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 2562e12e336a..dd256ec96e40 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index 1d677b5b4c92..fcd691c73617 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.23.0-rc-2 + 3.23.0-rc-3 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 10e98f550c33..999125b494d4 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 0194e661a17c..b1a90a021f30 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index 34165fe67f58..98694b7799dd 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.23.0-RC2 + 3.23.0-RC3 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index 7d9b6c0735ea..b8b94702a6e4 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC2 + 3.23.0-RC3 protobuf-java-util diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index f5c4c70d3a63..8b98114aba04 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.23.0RC2" +#define PHP_PROTOBUF_VERSION "3.23.0RC3" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 2f9a9f5d968e..6d0468046f50 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,6 +1,6 @@ """ Contains version numbers to be used in other bzl files """ -PROTOC_VERSION = "23.0-rc-2" -PROTOBUF_JAVA_VERSION = "3.23.0-RC2" -PROTOBUF_PYTHON_VERSION = "4.23.0rc2" -PROTOBUF_PHP_VERSION = "3.23.0RC2" -PROTOBUF_RUBY_VERSION = "3.23.0.rc.2" +PROTOC_VERSION = "23.0-rc-3" +PROTOBUF_JAVA_VERSION = "3.23.0-RC3" +PROTOBUF_PYTHON_VERSION = "4.23.0rc3" +PROTOBUF_PHP_VERSION = "3.23.0RC3" +PROTOBUF_RUBY_VERSION = "3.23.0.rc.3" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 47fc9b3421a4..71a283e69f77 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.23.0rc2' +__version__ = '4.23.0rc3' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 937ab3e25d3f..58ecfdaa4c8b 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.23.0.rc.2" + s.version = "3.23.0.rc.3" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index 0025923264cc..f615a9860f76 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.23.0-RC2 + 3.23.0-RC3 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.23.0-RC2 + 3.23.0-RC3 org.jruby diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index a14b89cfcb64..204e60140a2e 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -246,7 +246,7 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined #endif -#define PROTOBUF_VERSION_SUFFIX "-rc2" +#define PROTOBUF_VERSION_SUFFIX "-rc3" #if defined(PROTOBUF_NAMESPACE) || defined(PROTOBUF_NAMESPACE_ID) #error PROTOBUF_NAMESPACE or PROTOBUF_NAMESPACE_ID was previously defined diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 67d476ed3ecc..6f18ef2384ca 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -69,7 +69,7 @@ namespace internal { #define GOOGLE_PROTOBUF_VERSION 4023000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. -#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc2" +#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc3" // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code diff --git a/version.json b/version.json index 4ec814607468..359d07087f7e 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-dev", + "protoc_version": "23.0-rc3", "lts": false, - "date": "2023-04-28", + "date": "2023-05-05", "languages": { - "cpp": "4.23.0-dev", - "csharp": "3.23.0-dev", - "java": "3.23.0-dev", - "javascript": "3.23.0-dev", - "objectivec": "3.23.0-dev", - "php": "3.23.0-dev", - "python": "4.23.0-dev", - "ruby": "3.23.0-dev" + "cpp": "4.23.0-rc3", + "csharp": "3.23.0-rc3", + "java": "3.23.0-rc3", + "javascript": "3.23.0-rc3", + "objectivec": "3.23.0-rc3", + "php": "3.23.0-rc3", + "python": "4.23.0-rc3", + "ruby": "3.23.0-rc3" } } } \ No newline at end of file From 289eeaae58975bc6bbfcecebc0f0cd10c2e44f2c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 5 May 2023 09:17:00 -0700 Subject: [PATCH 456/463] Updating version.json to: 23.0-dev --- version.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/version.json b/version.json index 359d07087f7e..27230584dcdc 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-rc3", + "protoc_version": "23.0-dev", "lts": false, "date": "2023-05-05", "languages": { - "cpp": "4.23.0-rc3", - "csharp": "3.23.0-rc3", - "java": "3.23.0-rc3", - "javascript": "3.23.0-rc3", - "objectivec": "3.23.0-rc3", - "php": "3.23.0-rc3", - "python": "4.23.0-rc3", - "ruby": "3.23.0-rc3" + "cpp": "4.23.0-dev", + "csharp": "3.23.0-dev", + "java": "3.23.0-dev", + "javascript": "3.23.0-dev", + "objectivec": "3.23.0-dev", + "php": "3.23.0-dev", + "python": "4.23.0-dev", + "ruby": "3.23.0-dev" } } } \ No newline at end of file From 54caf40312b3e7fd7794e267ef17e3be202de83d Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 5 May 2023 16:24:31 -0700 Subject: [PATCH 457/463] Avoid using string(JOIN..., which requires cmake 3.12 This also downgrades our tests to use cmake 3.10, compatible with our support window. Closes #12672 PiperOrigin-RevId: 529840763 --- .github/workflows/test_cpp.yml | 16 ++++++++-------- cmake/install.cmake | 8 ++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 70f3a75a91dd..aae5df2c73e7 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -116,7 +116,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.10.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /test.sh ${{ matrix.flags}} ${{ env.CCACHE_CMAKE_FLAGS }} @@ -142,7 +142,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.10.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package \&\& @@ -171,16 +171,16 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7058879cac0224e443043eae2de82c13a63a54dd31855dac020c5522aa573cd2 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.10.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_BUILD_EXAMPLES=OFF \&\& - cd examples \&\& - mkdir build \&\& - cmake -S . -B build -DCMAKE_CXX_STANDARD=14 \&\& - cmake --build build + mkdir examples/build \&\& + cd examples/build \&\& + cmake .. -DCMAKE_CXX_STANDARD=14 \&\& + cmake --build . linux-cmake-submodules: name: Linux CMake Submodules @@ -200,7 +200,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e0eb6c69b7551d89f0dbdbe11906077a1d501229c28db39623b945e0c5d7029a + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /test.sh ${{ env.CCACHE_CMAKE_FLAGS }} diff --git a/cmake/install.cmake b/cmake/install.cmake index af891ca0c3d1..9d837ac536c9 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -6,8 +6,12 @@ foreach(_target IN LISTS protobuf_ABSL_USED_TARGETS) endforeach() list(APPEND _pc_targets "utf8_range") -string(JOIN " " _pc_target_list ${_pc_targets}) -set(_protobuf_PC_REQUIRES ${_pc_target_list}) +set(_protobuf_PC_REQUIRES "") +set(_sep "") +foreach (_target IN LISTS _pc_target_list) + string(CONCAT _protobuf_PC_REQUIRES "${_protobuf_PC_REQUIRES}" "${_sep}" "${_target}") + set(_sep " ") +endforeach () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf.pc.cmake ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc @ONLY) From bc9a1d219e9b0043036c6af612ae0e0de3369e0c Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 5 May 2023 21:25:58 -0700 Subject: [PATCH 458/463] Add tests for GCC support window. Note: gcc only supports docker images down to 9.5, and the 7.3 image is very old and problematic. A follow-up change might enable testing for GCC 7.3, which is our minimal supported version PiperOrigin-RevId: 529885733 --- .github/workflows/test_cpp.yml | 51 +++++++++++++++++++- src/google/protobuf/io/printer_death_test.cc | 1 + src/google/protobuf/port_def.inc | 6 +++ src/google/protobuf/repeated_field.h | 2 +- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index aae5df2c73e7..f4d1436ae195 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -51,6 +51,26 @@ jobs: bazel-cache: cpp_linux/${{ matrix.config.name }} bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} + linux-gcc: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + version: ['9.5', '13.1'] + name: Linux GCC ${{ matrix.version }} + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + - name: Run tests + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-5.4.0-2d15d9e888c9e7f90961dbd3afc8ea209717fb4b + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: cpp_linux/gcc-${{ matrix.version }} + bazel: test //pkg/... //src/... @com_google_protobuf_examples//... + linux-release: strategy: fail-fast: false # Don't cancel all jobs if one fails. @@ -122,7 +142,6 @@ jobs: /test.sh ${{ matrix.flags}} ${{ env.CCACHE_CMAKE_FLAGS }} -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package - linux-cmake-install: name: Linux CMake Install @@ -182,6 +201,36 @@ jobs: cmake .. -DCMAKE_CXX_STANDARD=14 \&\& cmake --build . + linux-cmake-gcc: + name: Linux CMake GCC + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + submodules: recursive + + - name: Setup ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 + with: + cache-prefix: linux-cmake-gcc + + - name: Run tests + uses: protocolbuffers/protobuf-ci/docker@v1 + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:13.1-5.4.0-307caa02808127e49720f3e77d6a9f3b3ef5a915 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + entrypoint: bash + command: >- + -c 'set -ex; + cd /workspace; + ccache -z; + cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}; + cmake --build . --parallel 20; + ctest --verbose --parallel 20; + ccache -s' + linux-cmake-submodules: name: Linux CMake Submodules runs-on: ubuntu-latest diff --git a/src/google/protobuf/io/printer_death_test.cc b/src/google/protobuf/io/printer_death_test.cc index 7365418273a7..992e3ca7efb1 100644 --- a/src/google/protobuf/io/printer_death_test.cc +++ b/src/google/protobuf/io/printer_death_test.cc @@ -90,6 +90,7 @@ class FakeAnnotationCollector : public AnnotationCollector { public: ~FakeAnnotationCollector() override = default; + using AnnotationCollector::AddAnnotation; void AddAnnotation(size_t begin_offset, size_t end_offset, const std::string& file_path, const std::vector& path) override { diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 204e60140a2e..a18fcc4912b7 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -1013,6 +1013,12 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and // This error has been generally flaky, but we need to disable it specifically // to fix https://github.com/protocolbuffers/protobuf/issues/12313 #pragma GCC diagnostic ignored "-Wunused-parameter" +#ifndef __clang__ +// This causes spurious warnings in GCC 13. +#pragma GCC diagnostic ignored "-Wstringop-overflow" +// This causes spurious warnings in GCC 13. +#pragma GCC diagnostic ignored "-Wself-move" +#endif #if __GNUC__ == 12 && __GNUC_MINOR__ < 4 // Wrong warning emitted when assigning a single char c-string to a std::string // in c++20 mode and optimization on. diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index d5e531a197cf..bfcb93833d08 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -977,7 +977,7 @@ PROTOBUF_NOINLINE void RepeatedField::GrowNoAnnotate(int current_size, Element* pold = elements(); // TODO(b/263791665): add absl::is_trivially_relocatable if (std::is_trivial::value) { - memcpy(pnew, pold, current_size * sizeof(Element)); + memcpy(static_cast(pnew), pold, current_size * sizeof(Element)); } else { for (Element* end = pnew + current_size; pnew != end; ++pnew, ++pold) { ::new (static_cast(pnew)) Element(std::move(*pold)); From 1ca4e9c4859a23112684138c78608ddc0b8f1770 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Sun, 7 May 2023 09:22:27 -0700 Subject: [PATCH 459/463] fix: missing -DPROTOBUF_USE_DLLS in pkg-config (#12700) When the protobuf libraries have been compiled as shared libraries the users of the library need to add `-DPROTOBUF_USE_DLLS` to their build line. Otherwise some symbols are missing. Fixes #12699 FWIW, I am not sure this is an ideal fix. It may be better to fix the headers such that no macros change the ABI. Closes #12700 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12700 from coryan:fix-define-protobuf-use-dlls-in-pkg-config-file 13c792eebd3d070fa25eff68cfca6ae1493e9503 PiperOrigin-RevId: 530116678 --- cmake/install.cmake | 4 ++++ cmake/protobuf-lite.pc.cmake | 2 +- cmake/protobuf.pc.cmake | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/install.cmake b/cmake/install.cmake index 9d837ac536c9..8c96981da8b3 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -12,6 +12,10 @@ foreach (_target IN LISTS _pc_target_list) string(CONCAT _protobuf_PC_REQUIRES "${_protobuf_PC_REQUIRES}" "${_sep}" "${_target}") set(_sep " ") endforeach () +set(_protobuf_PC_CFLAGS) +if (protobuf_BUILD_SHARED_LIBS) + set(_protobuf_PC_CFLAGS -DPROTOBUF_USE_DLLS) +endif () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf.pc.cmake ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc @ONLY) diff --git a/cmake/protobuf-lite.pc.cmake b/cmake/protobuf-lite.pc.cmake index 7f742c0258dc..eb5869e588f1 100644 --- a/cmake/protobuf-lite.pc.cmake +++ b/cmake/protobuf-lite.pc.cmake @@ -8,5 +8,5 @@ Description: Google's Data Interchange Format Version: @protobuf_VERSION@ Requires: @_protobuf_PC_REQUIRES@ Libs: -L${libdir} -lprotobuf-lite @CMAKE_THREAD_LIBS_INIT@ -Cflags: -I${includedir} +Cflags: -I${includedir} @_protobuf_PC_CFLAGS@ Conflicts: protobuf diff --git a/cmake/protobuf.pc.cmake b/cmake/protobuf.pc.cmake index 21bbf47fb1d1..81d87c1f7019 100644 --- a/cmake/protobuf.pc.cmake +++ b/cmake/protobuf.pc.cmake @@ -8,5 +8,5 @@ Description: Google's Data Interchange Format Version: @protobuf_VERSION@ Requires: @_protobuf_PC_REQUIRES@ Libs: -L${libdir} -lprotobuf @CMAKE_THREAD_LIBS_INIT@ -Cflags: -I${includedir} +Cflags: -I${includedir} @_protobuf_PC_CFLAGS@ Conflicts: protobuf-lite From 28c990508f398e6af9c26263dad600025cacb332 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Sun, 7 May 2023 09:23:49 -0700 Subject: [PATCH 460/463] fix: typo in `string(JOIN)` workaround (#12698) My sketch to fix #12672 was wrong. This works for realsies. Closes #12698 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12698 from coryan:fix-cmake-typo-in-string-join-workaround 182d2e248ee10403541c5ddd07857846a66ae57d PiperOrigin-RevId: 530116824 --- cmake/install.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.cmake b/cmake/install.cmake index 8c96981da8b3..e7eb2103be6e 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -8,7 +8,7 @@ list(APPEND _pc_targets "utf8_range") set(_protobuf_PC_REQUIRES "") set(_sep "") -foreach (_target IN LISTS _pc_target_list) +foreach (_target IN LISTS _pc_targets) string(CONCAT _protobuf_PC_REQUIRES "${_protobuf_PC_REQUIRES}" "${_sep}" "${_target}") set(_sep " ") endforeach () From fe1277f9a835d50285cc8e82df5a77daaee42d77 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Sun, 7 May 2023 12:43:50 -0700 Subject: [PATCH 461/463] fix: avoid warnings on Windows (#12701) On Wndows, `size_t` is 64-bits, and `int` is 32-bits. That makes conversions from `size_t` to `int` potentially lossy, and they generate warnings. In this case an `int` variable was assigned to `size_t` and then passed to functions consuming `int`. Seems simpler to use `auto` and avoid these problems altogether. Closes #12701 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12701 from coryan:fix-warnings-repeated-field-warnings-in-msvc b1ec34de77cbd31c914b810c87cbe16f1ce51a7d PiperOrigin-RevId: 530134611 --- src/google/protobuf/repeated_field.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index bfcb93833d08..057565a41652 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -494,7 +494,7 @@ template inline RepeatedField::RepeatedField(const RepeatedField& rhs) : current_size_(0), total_size_(0), arena_or_elements_(nullptr) { StaticValidityCheck(); - if (size_t size = rhs.current_size_) { + if (auto size = rhs.current_size_) { Grow(0, size); ExchangeCurrentSize(size); UninitializedCopyN(rhs.elements(), size, unsafe_elements()); @@ -775,7 +775,7 @@ inline void RepeatedField::Clear() { template inline void RepeatedField::MergeFrom(const RepeatedField& rhs) { ABSL_DCHECK_NE(&rhs, this); - if (size_t size = rhs.current_size_) { + if (auto size = rhs.current_size_) { Reserve(current_size_ + size); Element* dst = elements() + ExchangeCurrentSize(current_size_ + size); UninitializedCopyN(rhs.elements(), size, dst); From b880933ed39a3218176aa36a0fa109093b3b8e96 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Sun, 7 May 2023 13:12:01 -0700 Subject: [PATCH 462/463] fix: avoid warnings on MSVC (#12697) Warnings in header files can be a problem for consumers that enable `/WX` (or `-Werror`). In this case, using `... & -align` produces a warning (C4146) with MSVC. The fix is to use equivalent expression `... & ~(align - 1)`, which was already used in the same file. Fixes #12675 Closes #12697 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/12697 from coryan:fix-msvc-warnings-in-arena-align 835f3b489a85e1b50281ac2943bd74d159eb3ead PiperOrigin-RevId: 530137165 --- src/google/protobuf/arena_align.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/arena_align.h b/src/google/protobuf/arena_align.h index 958bb9d072b8..d63393c84311 100644 --- a/src/google/protobuf/arena_align.h +++ b/src/google/protobuf/arena_align.h @@ -99,7 +99,7 @@ struct ArenaAlignDefault { } static inline PROTOBUF_ALWAYS_INLINE constexpr size_t Ceil(size_t n) { - return (n + align - 1) & -align; + return (n + align - 1) & ~(align - 1); } static inline PROTOBUF_ALWAYS_INLINE constexpr size_t Floor(size_t n) { return (n & ~(align - 1)); @@ -113,7 +113,7 @@ struct ArenaAlignDefault { template static inline PROTOBUF_ALWAYS_INLINE T* Ceil(T* ptr) { uintptr_t intptr = reinterpret_cast(ptr); - return reinterpret_cast((intptr + align - 1) & -align); + return reinterpret_cast((intptr + align - 1) & ~(align - 1)); } template @@ -142,7 +142,9 @@ struct ArenaAlign { return (reinterpret_cast(ptr) & (align - 1)) == 0U; } - constexpr size_t Ceil(size_t n) const { return (n + align - 1) & -align; } + constexpr size_t Ceil(size_t n) const { + return (n + align - 1) & ~(align - 1); + } constexpr size_t Floor(size_t n) const { return (n & ~(align - 1)); } constexpr size_t Padded(size_t n) const { @@ -156,7 +158,7 @@ struct ArenaAlign { template T* Ceil(T* ptr) const { uintptr_t intptr = reinterpret_cast(ptr); - return reinterpret_cast((intptr + align - 1) & -align); + return reinterpret_cast((intptr + align - 1) & ~(align - 1)); } template From 6e511679de8ab0feefc1cdac1505b2fac5548e42 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 8 May 2023 09:48:50 -0700 Subject: [PATCH 463/463] Updating version.json and repo version numbers to: 23.0 --- CMakeLists.txt | 2 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/README.md | 6 +++--- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 2 +- java/util/pom.xml | 2 +- php/ext/google/protobuf/protobuf.h | 2 +- protobuf_version.bzl | 10 +++++----- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 ++-- src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/stubs/common.h | 2 +- version.json | 20 +++++++++---------- 23 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f2f0ba063e1..31e7883e944a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,7 +93,7 @@ if (protobuf_BUILD_SHARED_LIBS) endif () # Version metadata -set(protobuf_VERSION_STRING "4.23.0-rc-3") +set(protobuf_VERSION_STRING "4.23.0") set(protobuf_DESCRIPTION "Protocol Buffers") set(protobuf_CONTACT "protobuf@googlegroups.com") diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index a76d480c67df..6fb313c0093e 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '4.23.0-rc3' + s.version = '4.23.0' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index bbed94f833f4..acb38911d30d 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.23.0-rc3' + s.version = '3.23.0' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 99d8adf235ab..5dc2349bc707 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.23.0-rc3 + 3.23.0 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 6fb30ba2a119..032ccf784c58 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -5,7 +5,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.23.0-rc3 + 3.23.0 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/java/README.md b/java/README.md index fb7ed24a23fc..45205730a57b 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.23.0-rc-3 + 3.23.0 ``` @@ -37,14 +37,14 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.23.0-rc-3 + 3.23.0 ``` ### Gradle If you are using Gradle, add the following to your `build.gradle` file's -dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0-rc-3'` Again, +dependencies: `implementation 'com.google.protobuf:protobuf-java:3.23.0'` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 569b63651e64..d73be6f0e14c 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.23.0-RC3 + 3.23.0 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 8f5ece64a614..43903ddad2de 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 protobuf-java diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index 482ea7bfbd8f..4db88860915a 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index dd256ec96e40..f24abca75945 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index fcd691c73617..22563fe90d0c 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.23.0-rc-3 + 3.23.0 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 999125b494d4..ede4b4daa1de 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index b1a90a021f30..6e8c37849375 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index 98694b7799dd..8e6212efc585 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.23.0-RC3 + 3.23.0 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index b8b94702a6e4..54e98f213a8c 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.23.0-RC3 + 3.23.0 protobuf-java-util diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index 8b98114aba04..0879cf135c68 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.23.0RC3" +#define PHP_PROTOBUF_VERSION "3.23.0" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 6d0468046f50..9ee991feba6d 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,6 +1,6 @@ """ Contains version numbers to be used in other bzl files """ -PROTOC_VERSION = "23.0-rc-3" -PROTOBUF_JAVA_VERSION = "3.23.0-RC3" -PROTOBUF_PYTHON_VERSION = "4.23.0rc3" -PROTOBUF_PHP_VERSION = "3.23.0RC3" -PROTOBUF_RUBY_VERSION = "3.23.0.rc.3" +PROTOC_VERSION = "23.0" +PROTOBUF_JAVA_VERSION = "3.23.0" +PROTOBUF_PYTHON_VERSION = "4.23.0" +PROTOBUF_PHP_VERSION = "3.23.0" +PROTOBUF_RUBY_VERSION = "3.23.0" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 71a283e69f77..98a760614c44 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.23.0rc3' +__version__ = '4.23.0' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 58ecfdaa4c8b..14f334a0aecb 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.23.0.rc.3" + s.version = "3.23.0" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index f615a9860f76..52ae7149b8a6 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.23.0-RC3 + 3.23.0 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.23.0-RC3 + 3.23.0 org.jruby diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index a18fcc4912b7..4b61c3da653f 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -246,7 +246,7 @@ static_assert(PROTOBUF_CPLUSPLUS_MIN(201402L), "Protobuf only supports C++14 and #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined #endif -#define PROTOBUF_VERSION_SUFFIX "-rc3" +#define PROTOBUF_VERSION_SUFFIX "" #if defined(PROTOBUF_NAMESPACE) || defined(PROTOBUF_NAMESPACE_ID) #error PROTOBUF_NAMESPACE or PROTOBUF_NAMESPACE_ID was previously defined diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 6f18ef2384ca..61ea63543a68 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -69,7 +69,7 @@ namespace internal { #define GOOGLE_PROTOBUF_VERSION 4023000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. -#define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc3" +#define GOOGLE_PROTOBUF_VERSION_SUFFIX "" // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code diff --git a/version.json b/version.json index 27230584dcdc..a47f97cdef0b 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "23.x": { - "protoc_version": "23.0-dev", + "protoc_version": "23.0", "lts": false, - "date": "2023-05-05", + "date": "2023-05-08", "languages": { - "cpp": "4.23.0-dev", - "csharp": "3.23.0-dev", - "java": "3.23.0-dev", - "javascript": "3.23.0-dev", - "objectivec": "3.23.0-dev", - "php": "3.23.0-dev", - "python": "4.23.0-dev", - "ruby": "3.23.0-dev" + "cpp": "4.23.0", + "csharp": "3.23.0", + "java": "3.23.0", + "javascript": "3.23.0", + "objectivec": "3.23.0", + "php": "3.23.0", + "python": "4.23.0", + "ruby": "3.23.0" } } } \ No newline at end of file