Edition support (#1052)

This is mainly cl/611142449, cl/802300883, and a few other small cleanup CLs.

(This is not my code, just a sync of internal changes.)

Closes #945.

Author: osa1

protoc_plugin/CHANGELOG.md

@@ -2,13 +2,15 @@
 
 Note: this version requires protobuf 5.0.0.
 
+* Support protobuf editions. ([#1052])
 * Update generated code for protobuf 5.0.0.
 * Update generated `clone` members to take advantage of faster `deepCopy`
   implementation in protobuf 5.0.0. ([#742])
 * Code size improvements for enum fields. ([#1047])
 
 [#742]: https://github.com/google/protobuf.dart/pull/742
 [#1047]: https://github.com/google/protobuf.dart/pull/1047
+[#1052]: https://github.com/google/protobuf.dart/pull/1052
 
 ## 22.5.0
 

protoc_plugin/Makefile

@@ -39,6 +39,7 @@ TEST_PROTO_LIST = \
 	foo \
 	high_tagnumber \
 	import_clash \
+	import_option \
 	import_public \
 	json_name \
 	map_api \

protoc_plugin/lib/names.dart

@@ -117,8 +117,18 @@ String singleQuote(String input) {
 }
 
 /// Chooses the Dart name of an extension.
-String extensionName(FieldDescriptorProto descriptor, Set<String> usedNames) {
-  return _unusedMemberNames(descriptor, null, null, usedNames).fieldName;
+String extensionName(
+  FieldDescriptorProto descriptor,
+  Set<String> usedNames,
+  bool lowercaseGroupNames,
+) {
+  return _unusedMemberNames(
+    descriptor,
+    null,
+    null,
+    usedNames,
+    lowercaseGroupNames,
+  ).fieldName;
 }
 
 Iterable<String> extensionSuffixes() sync* {
@@ -281,6 +291,7 @@ MemberNames messageMemberNames(
   String parentClassName,
   Set<String> usedTopLevelNames, {
   Iterable<String> reserved = const [],
+  bool lowercaseGroupNames = false,
 }) {
   final fieldList = List<FieldDescriptorProto>.from(descriptor.field);
   final sourcePositions = fieldList.asMap().map(
@@ -340,7 +351,13 @@ MemberNames messageMemberNames(
       final index = indexes[field.name]!;
       final sourcePosition = sourcePositions[field.name];
       takeFieldNames(
-        _unusedMemberNames(field, index, sourcePosition, existingNames),
+        _unusedMemberNames(
+          field,
+          index,
+          sourcePosition,
+          existingNames,
+          lowercaseGroupNames,
+        ),
       );
     }
   }
@@ -470,14 +487,15 @@ FieldNames _unusedMemberNames(
   int? index,
   int? sourcePosition,
   Set<String> existingNames,
+  bool lowercaseGroupNames,
 ) {
   if (_isRepeated(field)) {
     return FieldNames(
       field,
       index,
       sourcePosition,
       disambiguateName(
-        _defaultFieldName(_fieldMethodSuffix(field)),
+        _defaultFieldName(_fieldMethodSuffix(field, lowercaseGroupNames)),
         existingNames,
         _memberNamesSuffix(field.number),
       ),
@@ -498,7 +516,7 @@ FieldNames _unusedMemberNames(
   }
 
   final name = disambiguateName(
-    _fieldMethodSuffix(field),
+    _fieldMethodSuffix(field, lowercaseGroupNames),
     existingNames,
     _memberNamesSuffix(field.number),
     generateVariants: generateNameVariants,
@@ -535,11 +553,15 @@ String _defaultEnsureMethodName(String fieldMethodSuffix) =>
 
 /// The suffix to use for this field in Dart method names.
 /// (It should be camelcase and begin with an uppercase letter.)
-String _fieldMethodSuffix(FieldDescriptorProto field) {
+String _fieldMethodSuffix(
+  FieldDescriptorProto field,
+  bool lowercaseGroupNames,
+) {
   var name = _nameOption(field)!;
   if (name.isNotEmpty) return _capitalize(name);
 
-  if (field.type != FieldDescriptorProto_Type.TYPE_GROUP) {
+  if (field.type != FieldDescriptorProto_Type.TYPE_GROUP ||
+      lowercaseGroupNames) {
     return underscoresToCamelCase(field.name);
   }
 

protoc_plugin/lib/protoc.dart

@@ -15,6 +15,7 @@ import 'src/gen/dart_options.pb.dart';
 import 'src/gen/google/api/client.pb.dart';
 import 'src/gen/google/protobuf/compiler/plugin.pb.dart';
 import 'src/gen/google/protobuf/descriptor.pb.dart';
+import 'src/gen/google/protobuf/dart_edition_defaults.pb.dart';
 import 'src/linker.dart';
 import 'src/options.dart';
 import 'src/output_config.dart';
@@ -34,3 +35,12 @@ part 'src/paths.dart';
 part 'src/protobuf_field.dart';
 part 'src/service_generator.dart';
 part 'src/well_known_types.dart';
+
+final FeatureSetDefaults pluginFeatureSetDefaults =
+    FeatureSetDefaults.fromBuffer(
+      base64Decode(ProtobufInternalDartEditionDefaults),
+    );
+
+const Edition pluginMinSupportedEdition = Edition.EDITION_PROTO2;
+
+const Edition pluginMaxSupportedEdition = Edition.EDITION_2024;

protoc_plugin/lib/src/base_type.dart

@@ -65,8 +65,13 @@ class BaseType {
   String getRepeatedDartTypeIterable(FileGenerator fileGen) =>
       '$coreImportPrefix.Iterable<${getDartType(fileGen)}>';
 
-  factory BaseType(FieldDescriptorProto field, GenerationContext ctx) {
+  factory BaseType(
+    FieldDescriptorProto field,
+    FeatureSet features,
+    GenerationContext ctx,
+  ) {
     String constSuffix;
+    FieldDescriptorProto_Type type;
 
     switch (field.type) {
       case FieldDescriptorProto_Type.TYPE_BOOL:
@@ -191,14 +196,17 @@ class BaseType {
         );
 
       case FieldDescriptorProto_Type.TYPE_GROUP:
-        constSuffix = 'G';
-        break;
       case FieldDescriptorProto_Type.TYPE_MESSAGE:
-        constSuffix = 'M';
-        break;
+        if (features.messageEncoding == FeatureSet_MessageEncoding.DELIMITED) {
+          constSuffix = 'G';
+          type = FieldDescriptorProto_Type.TYPE_GROUP;
+        } else {
+          constSuffix = 'M';
+          type = FieldDescriptorProto_Type.TYPE_MESSAGE;
+        }
       case FieldDescriptorProto_Type.TYPE_ENUM:
         constSuffix = 'E';
-        break;
+        type = FieldDescriptorProto_Type.TYPE_ENUM;
 
       default:
         throw ArgumentError('unimplemented type: ${field.type.name}');
@@ -210,7 +218,7 @@ class BaseType {
     }
 
     return BaseType._raw(
-      field.type,
+      type,
       constSuffix,
       generator.classname!,
       null,

protoc_plugin/lib/src/code_generator.dart

@@ -10,10 +10,16 @@ import 'package:fixnum/fixnum.dart';
 import 'package:protobuf/protobuf.dart';
 
 import '../names.dart' show lowerCaseFirstLetter;
-import '../protoc.dart' show FileGenerator;
+import '../protoc.dart'
+    show
+        FileGenerator,
+        pluginFeatureSetDefaults,
+        pluginMinSupportedEdition,
+        pluginMaxSupportedEdition;
 import 'gen/dart_options.pb.dart';
 import 'gen/google/api/client.pb.dart';
 import 'gen/google/protobuf/compiler/plugin.pb.dart';
+import 'gen/google/protobuf/descriptor.pb.dart';
 import 'linker.dart';
 import 'options.dart';
 import 'output_config.dart';
@@ -58,6 +64,8 @@ abstract class ProtobufContainer {
   // The generator containing this entity.
   ProtobufContainer? get parent;
 
+  FeatureSet get features;
+
   /// The top-level parent of this entity, or itself if it is a top-level
   /// entity.
   ProtobufContainer? get toplevelParent {
@@ -86,8 +94,8 @@ class CodeGenerator {
     Map<String, SingleOptionParser>? optionParsers,
     OutputConfiguration config = const DefaultOutputConfiguration(),
   }) async {
+    final editionDefaults = pluginFeatureSetDefaults;
     final extensions = ExtensionRegistry();
-
     Dart_options.registerAllExtensions(extensions);
     Client.registerAllExtensions(extensions);
 
@@ -118,7 +126,7 @@ class CodeGenerator {
     // (We may import it even if we don't generate the .pb.dart file.)
     final generators = <FileGenerator>[];
     for (final file in request.protoFile) {
-      generators.add(FileGenerator(file, options));
+      generators.add(FileGenerator(editionDefaults, file, options));
     }
 
     // Collect field types and importable files.
@@ -131,9 +139,12 @@ class CodeGenerator {
         response.file.addAll(gen.generateFiles(config));
       }
     }
-    response.supportedFeatures = Int64(
-      CodeGeneratorResponse_Feature.FEATURE_PROTO3_OPTIONAL.value,
-    );
+    response.supportedFeatures =
+        Int64(CodeGeneratorResponse_Feature.FEATURE_PROTO3_OPTIONAL.value) |
+        Int64(CodeGeneratorResponse_Feature.FEATURE_SUPPORTS_EDITIONS.value);
+    response.minimumEdition = pluginMinSupportedEdition.value;
+    response.maximumEdition = pluginMaxSupportedEdition.value;
+
     _streamOut.add(response.writeToBuffer());
   }
 }

protoc_plugin/lib/src/enum_generator.dart

@@ -14,6 +14,9 @@ class EnumGenerator extends ProtobufContainer {
   @override
   final ProtobufContainer parent;
 
+  @override
+  final FeatureSet features;
+
   @override
   final String classname;
 
@@ -50,7 +53,8 @@ class EnumGenerator extends ProtobufContainer {
           parent.fullName == ''
               ? descriptor.name
               : '${parent.fullName}.${descriptor.name}',
-      _descriptor = descriptor {
+      _descriptor = descriptor,
+      features = resolveFeatures(parent.features, descriptor.options.features) {
     final usedNames = {...reservedEnumNames};
     for (var i = 0; i < descriptor.value.length; i++) {
       final value = descriptor.value[i];

protoc_plugin/lib/src/extension_generator.dart

@@ -24,7 +24,7 @@ class ExtensionGenerator {
     Set<String> usedNames,
     int repeatedFieldIndex,
     int fieldIdTag,
-  ) : _extensionName = extensionName(_descriptor, usedNames),
+  ) : _extensionName = extensionName(_descriptor, usedNames, false),
       _fieldPathSegment = [fieldIdTag, repeatedFieldIndex];
 
   static const _topLevelFieldTag = 7;
@@ -71,6 +71,8 @@ class ExtensionGenerator {
   /// The generator of the .pb.dart file where this extension will be defined.
   FileGenerator? get fileGen => _parent.fileGen;
 
+  FeatureSet get features => _field.features;
+
   String get name {
     if (!_resolved) throw StateError('resolve not called');
     final name = _extensionName;

protoc_plugin/lib/src/file_generator.dart

@@ -18,8 +18,6 @@ const String _protobufImportUrl = 'package:protobuf/protobuf.dart';
 const String _typedDataImportPrefix = r'$typed_data';
 const String _typedDataImportUrl = 'dart:typed_data';
 
-enum ProtoSyntax { proto2, proto3 }
-
 /// Generates the Dart output files for one .proto input file.
 ///
 /// Outputs include .pb.dart, pbenum.dart, and .pbjson.dart.
@@ -140,14 +138,21 @@ class FileGenerator extends ProtobufContainer {
   /// Whether cross-references have been resolved.
   bool _linked = false;
 
-  final ProtoSyntax syntax;
+  final Edition edition;
 
-  FileGenerator(this.descriptor, this.options)
-    : protoFileUri = Uri.file(descriptor.name),
-      syntax =
-          descriptor.syntax == 'proto3'
-              ? ProtoSyntax.proto3
-              : ProtoSyntax.proto2 {
+  @override
+  final FeatureSet features;
+
+  FileGenerator(
+    FeatureSetDefaults editionDefaults,
+    this.descriptor,
+    this.options,
+  ) : protoFileUri = Uri.file(descriptor.name),
+      edition = _getEdition(descriptor),
+      features = resolveFeatures(
+        _getEditionDefaults(editionDefaults, _getEdition(descriptor)),
+        descriptor.options.features,
+      ) {
     if (protoFileUri.isAbsolute) {
       // protoc should never generate an import with an absolute path.
       throw 'FAILURE: Import with absolute path is not supported';
@@ -825,6 +830,51 @@ class ConditionalConstDefinition {
   }
 }
 
+Edition _getEdition(FileDescriptorProto file) {
+  if (file.edition != Edition.EDITION_UNKNOWN) {
+    return file.edition;
+  }
+  if (file.syntax == 'proto3') {
+    return Edition.EDITION_PROTO3;
+  }
+  return Edition.EDITION_PROTO2;
+}
+
+FeatureSet resolveFeatures(FeatureSet parent, FeatureSet child) {
+  final result = parent.deepCopy();
+  result.mergeFromMessage(child);
+  return result;
+}
+
+FeatureSet _getEditionDefaults(
+  FeatureSetDefaults editionDefaults,
+  Edition edition,
+) {
+  if (edition.value < editionDefaults.minimumEdition.value) {
+    throw ArgumentError(
+      'Edition $edition is earlier than the minimum supported edition ${editionDefaults.minimumEdition}!',
+    );
+  }
+  if (edition.value > editionDefaults.maximumEdition.value) {
+    throw ArgumentError(
+      'Edition $edition is later than the maximum supported edition ${editionDefaults.maximumEdition}!',
+    );
+  }
+  FeatureSetDefaults_FeatureSetEditionDefault? found;
+  for (final d in editionDefaults.defaults) {
+    if (d.edition.value > edition.value) {
+      break;
+    }
+    found = d;
+  }
+  if (found == null) {
+    throw ArgumentError('No default found for edition $edition!');
+  }
+  final defaults = found.fixedFeatures.deepCopy();
+  defaults.mergeFromMessage(found.overridableFeatures);
+  return defaults;
+}
+
 const _fileIgnores = {
   'annotate_overrides',
   'camel_case_types',

protoc_plugin/lib/src/gen/google/protobuf/dart_edition_defaults.pb.dart

@@ -0,0 +1,9 @@
+// Generated with third_party/dart/protoc_plugin/tool/regenerate.sh.
+// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// ignore_for_file: constant_identifier_names
+
+const ProtobufInternalDartEditionDefaults =
+    'ChcYhAciACoQCAEQAhgCIAMoATACOAJAAQoXGOcHIgAqEAgCEAEYASACKAEwATgCQAEKFxjoByIMCAEQARgBIAIoATABKgQ4AkABChcY6QciEAgBEAEYASACKAEwATgBQAIqACDmByjpBw==';