feat: add Markdown formatter

Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>

Author: yordis

lib/ex_doc/doc_ast.ex

@@ -65,6 +65,76 @@ defmodule ExDoc.DocAST do
     Enum.map(attrs, fn {key, val} -> " #{key}=\"#{ExDoc.Utils.h(val)}\"" end)
   end
 
+  @doc """
+  Transform AST into markdown string.
+
+  The optional `fun` argument allows post-processing each AST node
+  after it's been converted to markdown.
+  """
+  def to_markdown_string(ast, fun \\ fn _ast, string -> string end)
+
+  def to_markdown_string(binary, _fun) when is_binary(binary) do
+    ExDoc.Utils.h(binary)
+  end
+
+  def to_markdown_string(list, fun) when is_list(list) do
+    result = Enum.map_join(list, "", &to_markdown_string(&1, fun))
+    fun.(list, result)
+  end
+
+  def to_markdown_string({:comment, _attrs, inner, _meta} = ast, fun) do
+    fun.(ast, "<!--#{inner}-->")
+  end
+
+  def to_markdown_string({:code, attrs, inner, _meta} = ast, fun) do
+    lang = attrs[:class] || ""
+    result = """
+    ```#{lang}
+    #{inner}
+    ```
+    """
+
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({:a, attrs, inner, _meta} = ast, fun) do
+    result = "[#{to_markdown_string(inner, fun)}](#{attrs[:href]})"
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({:hr, _attrs, _inner, _meta} = ast, fun) do
+    result = "\n\n---\n\n"
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in [:p, :br] do
+    result = "\n\n"
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({:img, attrs, _inner, _meta} = ast, fun) do
+    alt = attrs[:alt] || ""
+    title = attrs[:title] || ""
+    result = "![#{alt}](#{attrs[:src]} \"#{title}\")"
+    fun.(ast, result)
+  end
+
+  # ignoring these: area base col command embed input keygen link meta param source track wbr
+  def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in @void_elements do
+    result = ""
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({_tag, _attrs, inner, %{verbatim: true}} = ast, fun) do
+    result = Enum.join(inner, "")
+    fun.(ast, result)
+  end
+
+  def to_markdown_string({_tag, _attrs, inner, _meta} = ast, fun) do
+    result = to_markdown_string(inner, fun)
+    fun.(ast, result)
+  end
+
   ## parse markdown
 
   defp parse_markdown(markdown, opts) do

lib/ex_doc/formatter.ex

@@ -48,14 +48,14 @@ defmodule ExDoc.Formatter do
 
                 specs = Enum.map(child_node.specs, &language.autolink_spec(&1, autolink_opts))
                 child_node = %{child_node | specs: specs}
-                render_doc(child_node, language, autolink_opts, opts)
+                render_doc(child_node, ext, language, autolink_opts, opts)
               end
 
-            %{render_doc(group, language, autolink_opts, opts) | docs: docs}
+            %{render_doc(group, ext, language, autolink_opts, opts) | docs: docs}
           end
 
         %{
-          render_doc(node, language, [{:id, node.id} | autolink_opts], opts)
+          render_doc(node, ext, language, [{:id, node.id} | autolink_opts], opts)
           | docs_groups: docs_groups
         }
       end,
@@ -117,11 +117,11 @@ defmodule ExDoc.Formatter do
 
   # Helper functions
 
-  defp render_doc(%{doc: nil} = node, _language, _autolink_opts, _opts),
+  defp render_doc(%{doc: nil} = node, _ext, _language, _autolink_opts, _opts),
     do: node
 
-  defp render_doc(%{doc: doc} = node, language, autolink_opts, opts) do
-    doc = autolink_and_highlight(doc, language, autolink_opts, opts)
+  defp render_doc(%{doc: doc} = node, ext, language, autolink_opts, opts) do
+    doc = autolink_and_render(doc, ext, language, autolink_opts, opts)
     %{node | doc: doc}
   end
 
@@ -137,7 +137,13 @@ defmodule ExDoc.Formatter do
     mod_id <> "." <> id
   end
 
-  defp autolink_and_highlight(doc, language, autolink_opts, opts) do
+  defp autolink_and_render(doc, ".md", language, autolink_opts, _opts) do
+    doc
+    |> language.autolink_doc(autolink_opts)
+    |> ExDoc.DocAST.to_markdown_string()
+  end
+
+  defp autolink_and_render(doc, _html_ext, language, autolink_opts, opts) do
     doc
     |> language.autolink_doc(autolink_opts)
     |> ExDoc.DocAST.highlight(language, opts)
@@ -183,6 +189,7 @@ defmodule ExDoc.Formatter do
     id = input_options[:filename] || input |> filename_to_title() |> Utils.text_to_id()
     source_file = input_options[:source] || input
     opts = [file: source_file, line: 1]
+    ext = Keyword.fetch!(autolink_opts, :ext)
 
     {extension, source, ast} =
       case extension_name(input) do
@@ -198,7 +205,7 @@ defmodule ExDoc.Formatter do
             source
             |> Markdown.to_ast(opts)
             |> ExDoc.DocAST.add_ids_to_headers([:h2, :h3])
-            |> autolink_and_highlight(language, [file: input] ++ autolink_opts, opts)
+            |> autolink_and_render(ext, language, [file: input] ++ autolink_opts, opts)
 
           {extension, source, ast}
 

lib/ex_doc/formatter/markdown.ex

@@ -0,0 +1,211 @@
+defmodule ExDoc.Formatter.MARKDOWN do
+  @moduledoc false
+
+  alias __MODULE__.{Templates}
+  alias ExDoc.Formatter
+  alias ExDoc.Utils
+
+  @doc """
+  Generates Markdown documentation for the given modules.
+  """
+  @spec run([ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()], ExDoc.Config.t()) :: String.t()
+  def run(project_nodes, filtered_modules, config) when is_map(config) do
+    Utils.unset_warned()
+
+    config = normalize_config(config)
+    File.rm_rf!(config.output)
+    File.mkdir_p!(config.output)
+
+    extras = Formatter.build_extras(config, ".md")
+
+    project_nodes =
+      project_nodes
+      |> Formatter.render_all(filtered_modules, ".md", config, highlight_tag: "samp")
+
+    nodes_map = %{
+      modules: Formatter.filter_list(:module, project_nodes),
+      tasks: Formatter.filter_list(:task, project_nodes)
+    }
+
+    config = %{config | extras: extras}
+
+    generate_nav(config, nodes_map)
+    generate_extras(config)
+    generate_list(config, nodes_map.modules)
+    generate_list(config, nodes_map.tasks)
+    generate_llm_index(config, nodes_map)
+
+    config.output |> Path.join("index.md") |> Path.relative_to_cwd()
+  end
+
+  defp normalize_config(config) do
+    output =
+      config.output
+      |> Path.expand()
+      |> Path.join("markdown")
+
+    %{config | output: output}
+  end
+
+  defp normalize_output(output) do
+    output
+    |> String.replace(~r/\r\n|\r|\n/, "\n")
+    |> String.replace(~r/\n{3,}/, "\n\n")
+  end
+
+  defp generate_nav(config, nodes) do
+    nodes =
+      Map.update!(nodes, :modules, fn modules ->
+        modules |> Enum.chunk_by(& &1.group) |> Enum.map(&{hd(&1).group, &1})
+      end)
+
+    content =
+      Templates.nav_template(config, nodes)
+      |> normalize_output()
+
+    File.write("#{config.output}/index.md", content)
+  end
+
+  defp generate_extras(config) do
+    for {_title, extras} <- config.extras do
+      Enum.each(extras, fn %{id: id, source: content} ->
+        output = "#{config.output}/#{id}.md"
+
+        if File.regular?(output) do
+          Utils.warn("file #{Path.relative_to_cwd(output)} already exists", [])
+        end
+
+        File.write!(output, normalize_output(content))
+      end)
+    end
+  end
+
+  defp generate_list(config, nodes) do
+    nodes
+    |> Task.async_stream(&generate_module_page(&1, config), timeout: :infinity)
+    |> Enum.map(&elem(&1, 1))
+  end
+
+  ## Helpers
+
+  defp generate_module_page(module_node, config) do
+    content =
+      Templates.module_page(config, module_node)
+      |> normalize_output()
+
+    File.write("#{config.output}/#{module_node.id}.md", content)
+  end
+
+  defp generate_llm_index(config, nodes_map) do
+    content = generate_llm_index_content(config, nodes_map)
+    File.write("#{config.output}/llms.txt", content)
+  end
+
+  defp generate_llm_index_content(config, nodes_map) do
+    project_info = """
+    # #{config.project} #{config.version}
+
+    #{config.project} documentation index for Large Language Models.
+
+    ## Modules
+
+    """
+
+    modules_info =
+      nodes_map.modules
+      |> Enum.map(fn module_node ->
+        "- **#{module_node.title}** (#{module_node.id}.md): #{module_node.doc |> extract_summary()}"
+      end)
+      |> Enum.join("\n")
+
+    tasks_info =
+      if length(nodes_map.tasks) > 0 do
+        tasks_list =
+          nodes_map.tasks
+          |> Enum.map(fn task_node ->
+            "- **#{task_node.title}** (#{task_node.id}.md): #{task_node.doc |> extract_summary()}"
+          end)
+          |> Enum.join("\n")
+
+        "\n\n## Mix Tasks\n\n" <> tasks_list
+      else
+        ""
+      end
+
+    extras_info =
+      if is_list(config.extras) and length(config.extras) > 0 do
+        extras_list =
+          config.extras
+          |> Enum.flat_map(fn
+            {_group, extras} when is_list(extras) -> extras
+            _ -> []
+          end)
+          |> Enum.map(fn extra ->
+            "- **#{extra.title}** (#{extra.id}.md): #{extra.title}"
+          end)
+          |> Enum.join("\n")
+
+        if extras_list == "" do
+          ""
+        else
+          "\n\n## Guides\n\n" <> extras_list
+        end
+      else
+        ""
+      end
+
+    project_info <> modules_info <> tasks_info <> extras_info
+  end
+
+  defp extract_summary(nil), do: "No documentation available"
+  defp extract_summary(""), do: "No documentation available"
+
+  defp extract_summary(doc) when is_binary(doc) do
+    doc
+    |> String.split("\n")
+    |> Enum.find("", fn line -> String.trim(line) != "" end)
+    |> String.trim()
+    |> case do
+      "" ->
+        "No documentation available"
+
+      summary ->
+        summary
+        |> String.slice(0, 150)
+        |> then(fn s -> if String.length(s) == 150, do: s <> "...", else: s end)
+    end
+  end
+
+  defp extract_summary(doc_ast) when is_list(doc_ast) do
+    # For DocAST (which is a list), extract the first text node
+    extract_first_text_from_ast(doc_ast)
+  end
+
+  defp extract_summary(_), do: "No documentation available"
+
+  defp extract_first_text_from_ast([]), do: "No documentation available"
+
+  defp extract_first_text_from_ast([{:p, _, content} | _rest]) do
+    extract_text_from_content(content)
+    |> String.slice(0, 150)
+    |> then(fn s -> if String.length(s) == 150, do: s <> "...", else: s end)
+  end
+
+  defp extract_first_text_from_ast([_node | rest]) do
+    extract_first_text_from_ast(rest)
+  end
+
+  defp extract_text_from_content([]), do: ""
+  defp extract_text_from_content([text | _rest]) when is_binary(text), do: text
+
+  defp extract_text_from_content([{_tag, _attrs, content} | rest]) do
+    case extract_text_from_content(content) do
+      "" -> extract_text_from_content(rest)
+      text -> text
+    end
+  end
+
+  defp extract_text_from_content([_node | rest]) do
+    extract_text_from_content(rest)
+  end
+end