Allow writing doc links to subsections using special characters

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -879,7 +879,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
                 // At this point we consider all articles with an H1 containing link "documentation extension" - some links might not resolve in the final documentation hierarchy
                 // and we will emit warnings for those later on when we finalize the bundle discovery phase.
                 if let link = result.value.title?.child(at: 0) as? AnyLink,
-                   let url = link.destination.flatMap(ValidatedURL.init) {
+                   let url = link.destination.flatMap(ValidatedURL.init(parsingExact:)) {
                     let reference = result.topicGraphNode.reference
 
                     let symbolPath = NodeURLGenerator.Path.documentation(path: url.components.path).stringValue

Sources/SwiftDocC/Infrastructure/DocumentationCurator.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -54,7 +54,7 @@ struct DocumentationCurator {
     /// Tries to resolve a link in the current module/context.
     mutating func referenceFromLink(link: Link, resolved: ResolvedTopicReference, source: URL?) -> ResolvedTopicReference? {
         // Try a link to a topic
-        guard let unresolved = link.destination.flatMap(ValidatedURL.init)?
+        guard let unresolved = link.destination.flatMap(ValidatedURL.init(parsingAuthoredLink:))?
             .requiring(scheme: ResolvedTopicReference.urlScheme)
             .map(UnresolvedTopicReference.init(topicURL:)) else {
                 // Emit a warning regarding the invalid link found in a task group.

Sources/SwiftDocC/Infrastructure/Symbol Link Resolution/AbsoluteSymbolLink.swift

@@ -46,7 +46,7 @@ public struct AbsoluteSymbolLink: CustomStringConvertible {
         // Begin by constructing a validated URL from the given string.
         // Normally symbol links would be validated with `init(symbolPath:)` but since this is expected
         // to be an absolute URL we parse it with `init(parsing:)` instead.
-        guard let validatedURL = ValidatedURL(parsing: string)?.requiring(scheme: ResolvedTopicReference.urlScheme) else {
+        guard let validatedURL = ValidatedURL(parsingExact: string)?.requiring(scheme: ResolvedTopicReference.urlScheme) else {
             return nil
         }
 

Sources/SwiftDocC/Model/DocumentationNode.swift

@@ -104,7 +104,7 @@ public struct DocumentationNode {
                 // so we can index all anchors found in the bundle for link resolution.
                 if let heading = child as? Heading, heading.level > 1, heading.level < 4 {
                     anchorSections.append(
-                        AnchorSection(reference: reference.withFragment(urlReadableFragment(heading.plainText)), title: heading.plainText)
+                        AnchorSection(reference: reference.withFragment(heading.plainText), title: heading.plainText)
                     )
                 }
             }

Sources/SwiftDocC/Model/Identifier.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -291,7 +291,7 @@ public struct ResolvedTopicReference: Hashable, Codable, Equatable, CustomString
         let newReference = ResolvedTopicReference(
             bundleIdentifier: bundleIdentifier,
             urlReadablePath: newPath,
-            urlReadableFragment: reference.fragment,
+            urlReadableFragment: reference.fragment.map(urlReadableFragment),
             sourceLanguages: sourceLanguages
         )
         return newReference
@@ -553,8 +553,11 @@ func urlReadablePath(_ path: String) -> String {
 }
 
 private extension CharacterSet {
-    static let invalidCharacterSet = CharacterSet(charactersIn: "'\"`")
-    static let whitespaceAndDashes = CharacterSet(charactersIn: "-").union(.whitespaces)
+    static let fragmentCharactersToRemove = CharacterSet.punctuationCharacters // Remove punctuation from fragments
+        .union(CharacterSet(charactersIn: "`"))       // Also consider back-ticks as punctuation. They are used as quotes around symbols or other code.
+        .subtracting(CharacterSet(charactersIn: "-")) // Don't remove hyphens. They are used as a whitespace replacement.
+    static let whitespaceAndDashes = CharacterSet.whitespaces
+        .union(CharacterSet(charactersIn: "-–—")) // hyphen, en dash, em dash
 }
 
 /// Creates a more readable version of a fragment by replacing characters that are not allowed in the fragment of a URL with hyphens.
@@ -572,7 +575,7 @@ func urlReadableFragment(_ fragment: String) -> String {
         .joined(separator: "-")
 
     // Remove invalid characters
-    fragment.unicodeScalars.removeAll(where: CharacterSet.invalidCharacterSet.contains)
+    fragment.unicodeScalars.removeAll(where: CharacterSet.fragmentCharactersToRemove.contains)
 
     return fragment
 }

Sources/SwiftDocC/Model/Rendering/RenderContentCompiler.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -95,7 +95,7 @@ struct RenderContentCompiler: MarkupVisitor {
             let externalLinkIdentifier = RenderReferenceIdentifier(forExternalLink: destination)
 
             if linkReferences.keys.contains(externalLinkIdentifier.identifier) {
-                // If we've already seen this link, return the existing reference with an overriden title.
+                // If we've already seen this link, return the existing reference with an overridden title.
                 return [RenderInlineContent.reference(identifier: externalLinkIdentifier,
                                                      isActive: true,
                                                      overridingTitle: plainTextLinkTitle.isEmpty ? nil : plainTextLinkTitle,
@@ -113,7 +113,7 @@ struct RenderContentCompiler: MarkupVisitor {
             }
         }
 
-        guard let unresolved = link.destination.flatMap(ValidatedURL.init)
+        guard let unresolved = link.destination.flatMap(ValidatedURL.init(parsingAuthoredLink:))
             .map({ UnresolvedTopicReference(topicURL: $0) }),
             // Try to resolve in the local context
             case let .success(resolved) = context.resolve(.unresolved(unresolved), in: identifier) else {

Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift

@@ -913,7 +913,7 @@ public struct RenderNodeTranslator: SemanticVisitor {
                     case let link as Link:
                         if !allowExternalLinks {
                             // For links require documentation scheme
-                            guard let _ = link.destination.flatMap(ValidatedURL.init)?.requiring(scheme: ResolvedTopicReference.urlScheme) else {
+                            guard let _ = link.destination.flatMap(ValidatedURL.init(parsingAuthoredLink:))?.requiring(scheme: ResolvedTopicReference.urlScheme) else {
                                 return nil
                             }
                         }

Sources/SwiftDocC/Semantics/ExternalLinks/ExternalMarkupReferenceWalker.swift

@@ -37,7 +37,7 @@ struct ExternalMarkupReferenceWalker: MarkupVisitor {
     mutating func visitLink(_ link: Link) {
         // Only process documentation links to external bundles
         guard let destination = link.destination,
-            let url = ValidatedURL(parsing: destination),
+            let url = ValidatedURL(parsingExact: destination),
             url.components.scheme == ResolvedTopicReference.urlScheme,
             let bundleID = url.components.host,
             bundleID != bundle.identifier else {

Sources/SwiftDocC/Semantics/MarkupReferenceResolver.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -96,7 +96,7 @@ struct MarkupReferenceResolver: MarkupRewriter {
         guard let destination = link.destination else {
             return link
         }
-        guard let url = ValidatedURL(parsing: destination) else {
+        guard let url = ValidatedURL(parsingAuthoredLink: destination) else {
             problems.append(invalidLinkDestinationProblem(destination: destination, source: source, range: link.range, severity: .warning))
             return link
         }
@@ -122,7 +122,7 @@ struct MarkupReferenceResolver: MarkupRewriter {
         }
 
         // We don't require a scheme here as the link can be a relative one, e.g. ``SwiftUI/View``.
-        let url = ValidatedURL(parsing: unresolvedDestination)?.requiring(scheme: ResolvedTopicReference.urlScheme) ?? ValidatedURL(symbolPath: unresolvedDestination)
+        let url = ValidatedURL(parsingExact: unresolvedDestination)?.requiring(scheme: ResolvedTopicReference.urlScheme) ?? ValidatedURL(symbolPath: unresolvedDestination)
         let unresolved = TopicReference.unresolved(.init(topicURL: url))
         guard let resolvedURL = resolve(reference: unresolved, range: range, severity: .warning, fromSymbolLink: true) else {
             return unresolvedDestination

Sources/SwiftDocC/Utility/ValidatedURL.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -37,13 +37,40 @@ public struct ValidatedURL: Hashable, Equatable {
     /// > URL with the "someMethodWithFirstValue" scheme which is a valid link but which won't resolve to the intended symbol.
     /// >
     /// > When working with symbol destinations use ``init(symbolPath:)`` instead.
-    init?(parsing string: String) {
+    /// >
+    /// > When working with authored documentation links use ``init(parsingAuthoredLink:)`` instead.
+    init?(parsingExact string: String) {
         guard let components = URLComponents(string: string) else {
             return nil
         }
         self.components = components
     }
 
+    /// Creates a new RFC 3986 valid URL by using the given string URL and percent escaping the fragment component if necessary.
+    ///
+    /// Will return `nil` when the given `string` is not a valid URL.
+    /// - Parameter string: Source URL address as string.
+    ///
+    /// If the parsed fragment component contains characters not allowed in the fragment of a URL, those characters will be percent encoded.
+    ///
+    /// Use this to parse author provided documentation links that may contain links to on-page subsections. Escaping the fragment allows authors
+    /// to write links to subsections using characters that wouldn't otherwise be allowed in a fragment of a URL.
+    init?(parsingAuthoredLink string: String) {
+        // Try to parse the string without escaping anything
+        if let parsed = ValidatedURL(parsingExact: string) {
+            self.components = parsed.components
+            return
+        }
+
+        // If the string doesn't contain a fragment and the string couldn't be parsed with `ValidatedURL(parsing:)` above, then consider it invalid.
+        guard let fragmentSeparatorIndex = string.firstIndex(of: "#"), var components = URLComponents(string: String(string[..<fragmentSeparatorIndex])) else {
+            return nil
+        }
+
+        components.percentEncodedFragment = String(string[fragmentSeparatorIndex...].dropFirst()).addingPercentEncoding(withAllowedCharacters: .urlFragmentAllowed)
+        self.components = components
+    }
+
     /// Creates a new RFC 3986 valid URL from the given URL.
     ///
     /// Will return `nil` when the given URL doesn't comply with RFC 3986.