Skip to content

Add protobuf code comments as Elixir module documentation #352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion lib/protobuf/protoc/context.ex
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ defmodule Protobuf.Protoc.Context do

### All files scope

# All parsed comments from the source file (mapping from path to comment)
# %{"1.4.2" => "this is a comment", "1.5.2.4.2" => "more comment\ndetails"}
comments: %{},

# Mapping from file name to (mapping from type name to metadata, like elixir type name)
# %{"example.proto" => %{".example.FooMsg" => %{type_name: "Example.FooMsg"}}}
global_type_mapping: %{},
Expand Down Expand Up @@ -42,7 +46,11 @@ defmodule Protobuf.Protoc.Context do
include_docs?: false,

# Elixirpb.FileOptions
custom_file_options: %{}
custom_file_options: %{},

# Current comment path. The locations are encoded into with a joining "."
# character. E.g. "4.2.3.0"
current_comment_path: ""

@spec custom_file_options_from_file_desc(t(), Google.Protobuf.FileDescriptorProto.t()) :: t()
def custom_file_options_from_file_desc(ctx, desc)
Expand All @@ -68,4 +76,17 @@ defmodule Protobuf.Protoc.Context do
module_prefix: Map.get(custom_file_opts, :module_prefix)
}
end

@doc """
Appends a comment path to the current comment path.

## Examples

iex> append_comment_path(%{current_comment_path: "4"}, "2.4")
%{current_comment_path: "4.2.4"}

"""
def append_comment_path(ctx, path) do
%{ctx | current_comment_path: String.trim(ctx.current_comment_path <> "." <> path, ".")}
end
end
20 changes: 17 additions & 3 deletions lib/protobuf/protoc/generator.ex
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,20 @@ defmodule Protobuf.Protoc.Generator do
ctx =
%Context{
ctx
| syntax: syntax(desc.syntax),
| comments: Protobuf.Protoc.Generator.Comment.parse(desc),
syntax: syntax(desc.syntax),
package: desc.package,
dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name)
}
|> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc)

enum_defmodules = Enum.map(desc.enum_type, &Generator.Enum.generate(ctx, &1))
enum_defmodules =
desc.enum_type
|> Enum.with_index()
|> Enum.map(fn {enum, index} ->
{Context.append_comment_path(ctx, "5.#{index}"), enum}
end)
|> Enum.map(fn {ctx, enum} -> Generator.Enum.generate(ctx, enum) end)

{nested_enum_defmodules, message_defmodules} =
Generator.Message.generate_list(ctx, desc.message_type)
Expand All @@ -51,7 +58,14 @@ defmodule Protobuf.Protoc.Generator do

service_defmodules =
if "grpc" in ctx.plugins do
Enum.map(desc.service, &Generator.Service.generate(ctx, &1))
desc.service
|> Enum.with_index()
|> Enum.map(fn {service, index} ->
Generator.Service.generate(
Context.append_comment_path(ctx, "6.#{index}"),
service
)
end)
else
[]
end
Expand Down
58 changes: 58 additions & 0 deletions lib/protobuf/protoc/generator/comment.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
defmodule Protobuf.Protoc.Generator.Comment do
@moduledoc false

alias Protobuf.Protoc.Context

@doc """
Parses comment information from `Google.Protobuf.FileDescriptorProto`
into a map with path keys.
"""
@spec parse(Google.Protobuf.FileDescriptorProto.t()) :: %{optional(String.t()) => String.t()}
def parse(file_descriptor_proto) do
file_descriptor_proto
|> get_locations()
|> Enum.reject(&empty_comment?/1)
|> Map.new(fn location ->
{Enum.join(location.path, "."), format_comment(location)}
end)
end

defp get_locations(%{source_code_info: %{location: value}}) when is_list(value),
do: value

defp get_locations(_value), do: []

defp empty_comment?(%{leading_comments: value}) when not is_nil(value) and value != "",
do: false

defp empty_comment?(%{trailing_comments: value}) when not is_nil(value) and value != "",
do: false

defp empty_comment?(%{leading_detached_comments: value}), do: Enum.empty?(value)

defp format_comment(location) do
[location.leading_comments, location.trailing_comments | location.leading_detached_comments]
|> Enum.reject(&is_nil/1)
|> Enum.map(&String.replace(&1, ~r/^\s*\*/, "", global: true))
|> Enum.join("\n\n")
|> String.replace(~r/\n{3,}/, "\n")
|> String.trim()
end

@doc """
Finds a comment via the context. Returns an empty string if the
comment is not found or if `include_docs?` is set to false.
"""
@spec get(Context.t()) :: String.t()
def get(%{include_docs?: false}), do: ""

def get(%{comments: comments, current_comment_path: path}),
do: get(comments, path)

@doc """
Finds a comment via a map of comments and a path. Returns an
empty string if the comment is not found
"""
@spec get(%{optional(String.t()) => String.t()}, String.t()) :: String.t()
def get(comments, path), do: Map.get(comments, path, "")
end
2 changes: 2 additions & 0 deletions lib/protobuf/protoc/generator/enum.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Enum do
@moduledoc false

alias Protobuf.Protoc.Context
alias Protobuf.Protoc.Generator.Comment
alias Protobuf.Protoc.Generator.Util

require EEx
Expand Down Expand Up @@ -34,6 +35,7 @@ defmodule Protobuf.Protoc.Generator.Enum do

content =
enum_template(
comment: Comment.get(ctx),
module: msg_name,
use_options: use_options,
fields: desc.value,
Expand Down
24 changes: 19 additions & 5 deletions lib/protobuf/protoc/generator/extension.ex
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ defmodule Protobuf.Protoc.Generator.Extension do

alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto, FileDescriptorProto}
alias Protobuf.Protoc.Context
alias Protobuf.Protoc.Generator.Comment
alias Protobuf.Protoc.Generator.Util

require EEx
Expand All @@ -29,7 +30,13 @@ defmodule Protobuf.Protoc.Generator.Extension do

module_contents =
Util.format(
extension_template(use_options: use_options, module: mod_name, extends: extensions)
extension_template(
comment: Comment.get(ctx),
use_options: use_options,
module: mod_name,
extends: extensions,
module_doc?: ctx.include_docs?
)
)

{mod_name, module_contents}
Expand Down Expand Up @@ -75,10 +82,15 @@ defmodule Protobuf.Protoc.Generator.Extension do
end

defp get_extensions_from_messages(%Context{} = ctx, use_options, descs) do
Enum.flat_map(descs, fn %DescriptorProto{} = desc ->
generate_module(ctx, use_options, desc) ++
descs
|> Enum.with_index()
|> Enum.flat_map(fn {desc, index} ->
generate_module(Context.append_comment_path(ctx, "7.#{index}"), use_options, desc) ++
get_extensions_from_messages(
%Context{ctx | namespace: ctx.namespace ++ [Macro.camelize(desc.name)]},
%Context{
Context.append_comment_path(ctx, "6.#{index}")
| namespace: ctx.namespace ++ [Macro.camelize(desc.name)]
},
use_options,
desc.nested_type
)
Expand All @@ -96,9 +108,11 @@ defmodule Protobuf.Protoc.Generator.Extension do
module_contents =
Util.format(
extension_template(
comment: Comment.get(ctx),
module: module_name,
use_options: use_options,
extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = ""))
extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = "")),
module_doc?: ctx.include_docs?
)
)

Expand Down
30 changes: 26 additions & 4 deletions lib/protobuf/protoc/generator/message.ex
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ defmodule Protobuf.Protoc.Generator.Message do
alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto}

alias Protobuf.Protoc.Context
alias Protobuf.Protoc.Generator.Comment
alias Protobuf.Protoc.Generator.Util
alias Protobuf.Protoc.Generator.Enum, as: EnumGenerator

Expand All @@ -21,7 +22,10 @@ defmodule Protobuf.Protoc.Generator.Message do
messages :: [{mod_name :: String.t(), contents :: String.t()}]}
def generate_list(%Context{} = ctx, descs) when is_list(descs) do
descs
|> Enum.map(fn desc -> generate(ctx, desc) end)
|> Enum.with_index()
|> Enum.map(fn {desc, index} ->
generate(Context.append_comment_path(ctx, "4.#{index}"), desc)
end)
|> Enum.unzip()
end

Expand All @@ -46,6 +50,7 @@ defmodule Protobuf.Protoc.Generator.Message do
{msg_name,
Util.format(
message_template(
comment: Comment.get(ctx),
module: msg_name,
use_options: msg_opts_str(ctx, desc.options),
oneofs: desc.oneof_decl,
Expand All @@ -61,11 +66,19 @@ defmodule Protobuf.Protoc.Generator.Message do
end

defp gen_nested_msgs(ctx, desc) do
Enum.map(desc.nested_type, fn msg_desc -> generate(ctx, msg_desc) end)
desc.nested_type
|> Enum.with_index()
|> Enum.map(fn {msg_desc, index} ->
generate(Context.append_comment_path(ctx, "3.#{index}"), msg_desc)
end)
end

defp gen_nested_enums(ctx, desc) do
Enum.map(desc.enum_type, fn enum_desc -> EnumGenerator.generate(ctx, enum_desc) end)
desc.enum_type
|> Enum.with_index()
|> Enum.map(fn {enum_desc, index} ->
EnumGenerator.generate(Context.append_comment_path(ctx, "4.#{index}"), enum_desc)
end)
end

defp gen_fields(syntax, fields) do
Expand Down Expand Up @@ -103,7 +116,15 @@ defmodule Protobuf.Protoc.Generator.Message do
oneofs = get_real_oneofs(desc.oneof_decl)

nested_maps = nested_maps(ctx, desc)
for field <- desc.field, do: get_field(ctx, field, nested_maps, oneofs)

for {field, index} <- Enum.with_index(desc.field) do
get_field(
Context.append_comment_path(ctx, "2.#{index}"),
field,
nested_maps,
oneofs
)
end
end

# Public and used by extensions.
Expand Down Expand Up @@ -137,6 +158,7 @@ defmodule Protobuf.Protoc.Generator.Message do

%{
name: field_desc.name,
comment: Comment.get(ctx),
number: field_desc.number,
label: label_name(field_desc.label),
type: type,
Expand Down
2 changes: 2 additions & 0 deletions lib/protobuf/protoc/generator/service.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Service do
@moduledoc false

alias Protobuf.Protoc.Context
alias Protobuf.Protoc.Generator.Comment
alias Protobuf.Protoc.Generator.Util

require EEx
Expand Down Expand Up @@ -31,6 +32,7 @@ defmodule Protobuf.Protoc.Generator.Service do
{mod_name,
Util.format(
service_template(
comment: Comment.get(ctx),
module: mod_name,
service_name: name,
package: ctx.package,
Expand Down
13 changes: 13 additions & 0 deletions lib/protobuf/protoc/generator/util.ex
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,19 @@ defmodule Protobuf.Protoc.Generator.Util do
|> IO.iodata_to_binary()
end

@spec pad_comment(String.t(), non_neg_integer()) :: String.t()
def pad_comment(comment, size) do
padding = String.duplicate(" ", size)

comment
|> String.split("\n")
|> Enum.map(fn line ->
trimmed = String.trim_leading(line, " ")
padding <> trimmed
end)
|> Enum.join("\n")
end

@spec version() :: String.t()
def version do
{:ok, value} = :application.get_key(:protobuf, :vsn)
Expand Down
10 changes: 5 additions & 5 deletions mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -149,15 +149,15 @@ defmodule Protobuf.Mixfile do
proto_src = path_in_protobuf_source(["src"])

protoc!(
"-I #{proto_src} -I src -I test/protobuf/protoc/proto",
"-I #{proto_src} -I src -I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
"./generated",
["test/protobuf/protoc/proto/extension.proto"]
)

protoc!(
"-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my",
"-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my,include_docs=true",
"./generated",
["test/protobuf/protoc/proto/test.proto"]
["test/protobuf/protoc/proto/test.proto", "test/protobuf/protoc/proto/service.proto"]
)

protoc!(
Expand All @@ -168,7 +168,7 @@ defmodule Protobuf.Mixfile do
)

protoc!(
"-I test/protobuf/protoc/proto",
"-I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
"./generated",
["test/protobuf/protoc/proto/no_package.proto"]
)
Expand All @@ -194,7 +194,7 @@ defmodule Protobuf.Mixfile do
google/protobuf/test_messages_proto3.proto
)

protoc!("-I \"#{proto_root}\"", "./generated", files)
protoc!("-I \"#{proto_root}\" --elixir_opt=include_docs=true", "./generated", files)
end

defp gen_conformance_protos(_args) do
Expand Down
9 changes: 8 additions & 1 deletion priv/templates/enum.ex.eex
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
defmodule <%= @module %> do
<%= unless @module_doc? do %>
<%= if @module_doc? do %>
<%= if @comment != "" do %>
@moduledoc """
<%= Protobuf.Protoc.Generator.Util.pad_comment(@comment, 2) %>
"""
<% end %>
<% else %>
@moduledoc false
<% end %>

use Protobuf, <%= @use_options %>

<%= if @descriptor_fun_body do %>
Expand Down
9 changes: 9 additions & 0 deletions priv/templates/extension.ex.eex
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
defmodule <%= @module %> do
<%= if @module_doc? do %>
<%= if @comment != "" do %>
@moduledoc """
<%= Protobuf.Protoc.Generator.Util.pad_comment(@comment, 2) %>
"""
<% end %>
<% else %>
@moduledoc false
<% end %>

use Protobuf, <%= @use_options %>

<% if @extends == [], do: raise("Fuck! #{@module}") %>
Expand Down
Loading
Loading