Skip to content

Commit

Permalink
Auto merge of rust-lang#80550 - bugadani:markdown-refactor, r=jyn514
Browse files Browse the repository at this point in the history 
Cleanup markdown span handling, take 2

This PR includes the cleanups made in rust-lang#80244 except for the removal of `locate()`.

While the biggest conceptual part in rust-lang#80244 was the removal of `locate()`, it introduced a diagnostic regression.

Additional cleanup:
 - Use `RefCell` to avoid building two separate vectors for the links

Work to do:
- [ ] Decide if `locate()` can be simplified by assuming `s` is always in `md`
- [ ] Should probably add some tests that still provide the undesired diagnostics causing rust-lang#80381

cc `@jyn514` This is the best I can do without patching Pulldown to provide multiple ranges for reference-style links. Also, since `locate` is probably more efficient than `rfind` (at least it's constant time), I decided to not check the link type and just cover every &str as it was before.
  • Loading branch information
@bors
bors committed Jan 2, 2021
2 parents 90ccf4f + f073543 commit fd85ca0
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 103 deletions.
155 changes: 84 additions & 71 deletions src/librustdoc/html/markdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use rustc_session::lint;
use rustc_span::edition::Edition;
use rustc_span::Span;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::VecDeque;
use std::default::Default;
use std::fmt::Write;
Expand Down Expand Up @@ -414,11 +415,13 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for LinkReplacer<'a, I> {
}
}

type SpannedEvent<'a> = (Event<'a>, Range<usize>);

/// Make headings links with anchor IDs and build up TOC.
struct HeadingLinks<'a, 'b, 'ids, I> {
inner: I,
toc: Option<&'b mut TocBuilder>,
buf: VecDeque<Event<'a>>,
buf: VecDeque<SpannedEvent<'a>>,
id_map: &'ids mut IdMap,
}

Expand All @@ -428,48 +431,48 @@ impl<'a, 'b, 'ids, I> HeadingLinks<'a, 'b, 'ids, I> {
}
}

impl<'a, 'b, 'ids, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a, 'b, 'ids, I> {
type Item = Event<'a>;
impl<'a, 'b, 'ids, I: Iterator<Item = SpannedEvent<'a>>> Iterator
for HeadingLinks<'a, 'b, 'ids, I>
{
type Item = SpannedEvent<'a>;

fn next(&mut self) -> Option<Self::Item> {
if let Some(e) = self.buf.pop_front() {
return Some(e);
}

let event = self.inner.next();
if let Some(Event::Start(Tag::Heading(level))) = event {
if let Some((Event::Start(Tag::Heading(level)), _)) = event {
let mut id = String::new();
for event in &mut self.inner {
match &event {
match &event.0 {
Event::End(Tag::Heading(..)) => break,
Event::Start(Tag::Link(_, _, _)) | Event::End(Tag::Link(..)) => {}
Event::Text(text) | Event::Code(text) => {
id.extend(text.chars().filter_map(slugify));
self.buf.push_back(event);
}
_ => {}
}
match event {
Event::Start(Tag::Link(_, _, _)) | Event::End(Tag::Link(..)) => {}
event => self.buf.push_back(event),
_ => self.buf.push_back(event),
}
}
let id = self.id_map.derive(id);

if let Some(ref mut builder) = self.toc {
let mut html_header = String::new();
html::push_html(&mut html_header, self.buf.iter().cloned());
html::push_html(&mut html_header, self.buf.iter().map(|(ev, _)| ev.clone()));
let sec = builder.push(level as u32, html_header, id.clone());
self.buf.push_front(Event::Html(format!("{} ", sec).into()));
self.buf.push_front((Event::Html(format!("{} ", sec).into()), 0..0));
}

self.buf.push_back(Event::Html(format!("</a></h{}>", level).into()));
self.buf.push_back((Event::Html(format!("</a></h{}>", level).into()), 0..0));

let start_tags = format!(
"<h{level} id=\"{id}\" class=\"section-header\">\
<a href=\"#{id}\">",
id = id,
level = level
);
return Some(Event::Html(start_tags.into()));
return Some((Event::Html(start_tags.into()), 0..0));
}
event
}
Expand Down Expand Up @@ -555,23 +558,23 @@ impl<'a, I> Footnotes<'a, I> {
}
}

impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
type Item = Event<'a>;
impl<'a, I: Iterator<Item = SpannedEvent<'a>>> Iterator for Footnotes<'a, I> {
type Item = SpannedEvent<'a>;

fn next(&mut self) -> Option<Self::Item> {
loop {
match self.inner.next() {
Some(Event::FootnoteReference(ref reference)) => {
Some((Event::FootnoteReference(ref reference), range)) => {
let entry = self.get_entry(&reference);
let reference = format!(
"<sup id=\"fnref{0}\"><a href=\"#fn{0}\">{0}</a></sup>",
(*entry).1
);
return Some(Event::Html(reference.into()));
return Some((Event::Html(reference.into()), range));
}
Some(Event::Start(Tag::FootnoteDefinition(def))) => {
Some((Event::Start(Tag::FootnoteDefinition(def)), _)) => {
let mut content = Vec::new();
for event in &mut self.inner {
for (event, _) in &mut self.inner {
if let Event::End(Tag::FootnoteDefinition(..)) = event {
break;
}
Expand Down Expand Up @@ -602,7 +605,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
ret.push_str("</li>");
}
ret.push_str("</ol></div>");
return Some(Event::Html(ret.into()));
return Some((Event::Html(ret.into()), 0..0));
} else {
return None;
}
Expand Down Expand Up @@ -912,13 +915,14 @@ impl Markdown<'_> {
};

let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut replacer));
let p = p.into_offset_iter();

let mut s = String::with_capacity(md.len() * 3 / 2);

let p = HeadingLinks::new(p, None, &mut ids);
let p = LinkReplacer::new(p, links);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = LinkReplacer::new(p.map(|(ev, _)| ev), links);
let p = CodeBlocks::new(p, codes, edition, playground);
html::push_html(&mut s, p);

s
Expand All @@ -929,16 +933,16 @@ impl MarkdownWithToc<'_> {
crate fn into_string(self) -> String {
let MarkdownWithToc(md, mut ids, codes, edition, playground) = self;

let p = Parser::new_ext(md, opts());
let p = Parser::new_ext(md, opts()).into_offset_iter();

let mut s = String::with_capacity(md.len() * 3 / 2);

let mut toc = TocBuilder::new();

{
let p = HeadingLinks::new(p, Some(&mut toc), &mut ids);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = CodeBlocks::new(p.map(|(ev, _)| ev), codes, edition, playground);
html::push_html(&mut s, p);
}

Expand All @@ -954,19 +958,19 @@ impl MarkdownHtml<'_> {
if md.is_empty() {
return String::new();
}
let p = Parser::new_ext(md, opts());
let p = Parser::new_ext(md, opts()).into_offset_iter();

// Treat inline HTML as plain text.
let p = p.map(|event| match event {
Event::Html(text) => Event::Text(text),
let p = p.map(|event| match event.0 {
Event::Html(text) => (Event::Text(text), event.1),
_ => event,
});

let mut s = String::with_capacity(md.len() * 3 / 2);

let p = HeadingLinks::new(p, None, &mut ids);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = CodeBlocks::new(p.map(|(ev, _)| ev), codes, edition, playground);
html::push_html(&mut s, p);

s
Expand Down Expand Up @@ -1119,56 +1123,65 @@ crate fn plain_text_summary(md: &str) -> String {
s
}

crate fn markdown_links(md: &str) -> Vec<(String, Option<Range<usize>>)> {
crate fn markdown_links(md: &str) -> Vec<(String, Range<usize>)> {
if md.is_empty() {
return vec![];
}

let mut links = vec![];
let mut shortcut_links = vec![];

{
let locate = |s: &str| unsafe {
let s_start = s.as_ptr();
let s_end = s_start.add(s.len());
let md_start = md.as_ptr();
let md_end = md_start.add(md.len());
if md_start <= s_start && s_end <= md_end {
let start = s_start.offset_from(md_start) as usize;
let end = s_end.offset_from(md_start) as usize;
Some(start..end)
} else {
None
}
};
let links = RefCell::new(vec![]);

// FIXME: remove this function once pulldown_cmark can provide spans for link definitions.
let locate = |s: &str, fallback: Range<usize>| unsafe {
let s_start = s.as_ptr();
let s_end = s_start.add(s.len());
let md_start = md.as_ptr();
let md_end = md_start.add(md.len());
if md_start <= s_start && s_end <= md_end {
let start = s_start.offset_from(md_start) as usize;
let end = s_end.offset_from(md_start) as usize;
start..end
} else {
fallback
}
};

let span_for_link = |link: &CowStr<'_>, span: Range<usize>| {
// For diagnostics, we want to underline the link's definition but `span` will point at
// where the link is used. This is a problem for reference-style links, where the definition
// is separate from the usage.
match link {
// `Borrowed` variant means the string (the link's destination) may come directly from
// the markdown text and we can locate the original link destination.
// NOTE: LinkReplacer also provides `Borrowed` but possibly from other sources,
// so `locate()` can fall back to use `span`.
CowStr::Borrowed(s) => locate(s, span),

// For anything else, we can only use the provided range.
CowStr::Boxed(_) | CowStr::Inlined(_) => span,
}
};

let mut push = |link: BrokenLink<'_>| {
// FIXME: use `link.span` instead of `locate`
// (doing it now includes the `[]` as well as the text)
shortcut_links.push((link.reference.to_owned(), locate(link.reference)));
None
};
let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut push));

// There's no need to thread an IdMap through to here because
// the IDs generated aren't going to be emitted anywhere.
let mut ids = IdMap::new();
let iter = Footnotes::new(HeadingLinks::new(p, None, &mut ids));

for ev in iter {
if let Event::Start(Tag::Link(_, dest, _)) = ev {
debug!("found link: {}", dest);
links.push(match dest {
CowStr::Borrowed(s) => (s.to_owned(), locate(s)),
s @ (CowStr::Boxed(..) | CowStr::Inlined(..)) => (s.into_string(), None),
});
}
let mut push = |link: BrokenLink<'_>| {
let span = span_for_link(&CowStr::Borrowed(link.reference), link.span);
links.borrow_mut().push((link.reference.to_owned(), span));
None
};
let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut push)).into_offset_iter();

// There's no need to thread an IdMap through to here because
// the IDs generated aren't going to be emitted anywhere.
let mut ids = IdMap::new();
let iter = Footnotes::new(HeadingLinks::new(p, None, &mut ids));

for ev in iter {
if let Event::Start(Tag::Link(_, dest, _)) = ev.0 {
debug!("found link: {}", dest);
let span = span_for_link(&dest, ev.1);
links.borrow_mut().push((dest.into_string(), span));
}
}

links.append(&mut shortcut_links);

links
links.into_inner()
}

#[derive(Debug)]
Expand Down
Loading

0 comments on commit fd85ca0

Please sign in to comment.