Skip to content

Generalize the removal of compiler argument options during indexing #1314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,13 @@ let package = Package(
exclude: ["CMakeLists.txt"]
),

.testTarget(
name: "SemanticIndexTests",
dependencies: [
"SemanticIndex"
]
),

// MARK: SKCore
// Data structures and algorithms useful across the project, but not necessarily
// suitable for use in other packages.
Expand Down
1 change: 1 addition & 0 deletions Sources/SemanticIndex/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

add_library(SemanticIndex STATIC
CheckedIndex.swift
CompilerCommandLineOption.swift
IndexTaskDescription.swift
PreparationTaskDescription.swift
SemanticIndexManager.swift
Expand Down
128 changes: 128 additions & 0 deletions Sources/SemanticIndex/CompilerCommandLineOption.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

@_spi(Testing) public struct CompilerCommandLineOption {
/// Return value of `matches(argument:)`.
public enum Match {
/// The `CompilerCommandLineOption` matched the command line argument. The next element in the command line is a
/// separate argument and should not be removed.
case removeOption

/// The `CompilerCommandLineOption` matched the command line argument. The next element in the command line is an
/// argument to this option and should be removed as well.
case removeOptionAndNextArgument
}

public enum DashSpelling {
case singleDash
case doubleDash
}

public enum ArgumentStyles {
/// A command line option where arguments can be passed without a space such as `-MT/file.txt`.
case noSpace
/// A command line option where the argument is passed, separated by a space (eg. `--serialize-diagnostics /file.txt`)
case separatedBySpace
/// A command line option where the argument is passed after a `=`, eg. `-fbuild-session-file=`.
case separatedByEqualSign
}

/// The name of the option, without any preceeding `-` or `--`.
private let name: String

/// Whether the option can be spelled with one or two dashes.
private let dashSpellings: [DashSpelling]

/// The ways that arguments can specified after the option. Empty if the option is a flag that doesn't take any
/// argument.
private let argumentStyles: [ArgumentStyles]

public static func flag(_ name: String, _ dashSpellings: [DashSpelling]) -> CompilerCommandLineOption {
precondition(!dashSpellings.isEmpty)
return CompilerCommandLineOption(name: name, dashSpellings: dashSpellings, argumentStyles: [])
}

public static func option(
_ name: String,
_ dashSpellings: [DashSpelling],
_ argumentStyles: [ArgumentStyles]
) -> CompilerCommandLineOption {
precondition(!dashSpellings.isEmpty)
precondition(!argumentStyles.isEmpty)
return CompilerCommandLineOption(name: name, dashSpellings: dashSpellings, argumentStyles: argumentStyles)
}

public func matches(argument: String) -> Match? {
let argumentName: Substring
if argument.hasPrefix("--") {
if dashSpellings.contains(.doubleDash) {
argumentName = argument.dropFirst(2)
} else {
return nil
}
} else if argument.hasPrefix("-") {
if dashSpellings.contains(.singleDash) {
argumentName = argument.dropFirst(1)
} else {
return nil
}
} else {
return nil
}
guard argumentName.hasPrefix(self.name) else {
// Fast path in case the argument doesn't match.
return nil
}

// Examples:
// - self.name: "emit-module", argument: "-emit-module", then textAfterArgumentName: ""
// - self.name: "o", argument: "-o", then textAfterArgumentName: ""
// - self.name: "o", argument: "-output-file-map", then textAfterArgumentName: "utput-file-map"
// - self.name: "MT", argument: "-MT/path/to/depfile", then textAfterArgumentName: "/path/to/depfile"
// - self.name: "fbuild-session-file", argument: "-fbuild-session-file=/path/to/file", then textAfterArgumentName: "=/path/to/file"
let textAfterArgumentName: Substring = argumentName.dropFirst(self.name.count)

if argumentStyles.isEmpty {
if textAfterArgumentName.isEmpty {
return .removeOption
}
// The command line option is a flag but there is text remaining after the argument name. Thus the flag didn't
// match. Eg. self.name: "o" and argument: "-output-file-map"
return nil
}

for argumentStyle in argumentStyles {
switch argumentStyle {
case .noSpace where !textAfterArgumentName.isEmpty:
return .removeOption
case .separatedBySpace where textAfterArgumentName.isEmpty:
return .removeOptionAndNextArgument
case .separatedByEqualSign where textAfterArgumentName.hasPrefix("="):
return .removeOption
default:
break
}
}
return nil
}
}

extension Array<CompilerCommandLineOption> {
func firstMatch(for argument: String) -> CompilerCommandLineOption.Match? {
for optionToRemove in self {
if let match = optionToRemove.matches(argument: argument) {
return match
}
}
return nil
}
}
112 changes: 59 additions & 53 deletions Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift
Original file line number Diff line number Diff line change
Expand Up @@ -281,37 +281,38 @@ private func adjustSwiftCompilerArgumentsForIndexStoreUpdate(
_ compilerArguments: [String],
fileToIndex: DocumentURI
) -> [String] {
let removeFlags: Set<String> = [
"-c",
"-disable-cmo",
"-emit-dependencies",
"-emit-module-interface",
"-emit-module",
"-emit-module",
"-emit-objc-header",
"-incremental",
"-no-color-diagnostics",
"-parseable-output",
"-save-temps",
"-serialize-diagnostics",
"-use-frontend-parseable-output",
"-validate-clang-modules-once",
"-whole-module-optimization",
]
let optionsToRemove: [CompilerCommandLineOption] = [
.flag("c", [.singleDash]),
.flag("disable-cmo", [.singleDash]),
.flag("emit-dependencies", [.singleDash]),
.flag("emit-module-interface", [.singleDash]),
.flag("emit-module", [.singleDash]),
.flag("emit-objc-header", [.singleDash]),
.flag("incremental", [.singleDash]),
.flag("no-color-diagnostics", [.singleDash]),
.flag("parseable-output", [.singleDash]),
.flag("save-temps", [.singleDash]),
.flag("serialize-diagnostics", [.singleDash]),
.flag("use-frontend-parseable-output", [.singleDash]),
.flag("validate-clang-modules-once", [.singleDash]),
.flag("whole-module-optimization", [.singleDash]),

let removeArguments: Set<String> = [
"-clang-build-session-file",
"-emit-module-interface-path",
"-emit-module-path",
"-emit-objc-header-path",
"-emit-package-module-interface-path",
"-emit-private-module-interface-path",
"-num-threads",
"-o",
"-output-file-map",
.option("clang-build-session-file", [.singleDash], [.separatedBySpace]),
.option("emit-module-interface-path", [.singleDash], [.separatedBySpace]),
.option("emit-module-path", [.singleDash], [.separatedBySpace]),
.option("emit-objc-header-path", [.singleDash], [.separatedBySpace]),
.option("emit-package-module-interface-path", [.singleDash], [.separatedBySpace]),
.option("emit-private-module-interface-path", [.singleDash], [.separatedBySpace]),
.option("num-threads", [.singleDash], [.separatedBySpace]),
// Technically, `-o` and the output file don't need to be separated by a space. Eg. `swiftc -oa file.swift` is
// valid and will write to an output file named `a`.
// We can't support that because the only way to know that `-output-file-map` is a different flag and not an option
// to write to an output file named `utput-file-map` is to know all compiler arguments of `swiftc`, which we don't.
.option("o", [.singleDash], [.separatedBySpace]),
.option("output-file-map", [.singleDash], [.separatedBySpace, .separatedByEqualSign]),
]

let removeFrontendFlags: Set<String> = [
let removeFrontendFlags = [
"-experimental-skip-non-inlinable-function-bodies",
"-experimental-skip-all-function-bodies",
]
Expand All @@ -320,12 +321,14 @@ private func adjustSwiftCompilerArgumentsForIndexStoreUpdate(
result.reserveCapacity(compilerArguments.count)
var iterator = compilerArguments.makeIterator()
while let argument = iterator.next() {
if removeFlags.contains(argument) {
switch optionsToRemove.firstMatch(for: argument) {
case .removeOption:
continue
}
if removeArguments.contains(argument) {
case .removeOptionAndNextArgument:
_ = iterator.next()
continue
case nil:
break
}
if argument == "-Xfrontend" {
if let nextArgument = iterator.next() {
Expand Down Expand Up @@ -356,43 +359,46 @@ private func adjustClangCompilerArgumentsForIndexStoreUpdate(
_ compilerArguments: [String],
fileToIndex: DocumentURI
) -> [String] {
let removeFlags: Set<String> = [
let optionsToRemove: [CompilerCommandLineOption] = [
// Disable writing of a depfile
"-M",
"-MD",
"-MMD",
"-MG",
"-MM",
"-MV",
.flag("M", [.singleDash]),
.flag("MD", [.singleDash]),
.flag("MMD", [.singleDash]),
.flag("MG", [.singleDash]),
.flag("MM", [.singleDash]),
.flag("MV", [.singleDash]),
// Don't create phony targets
"-MP",
// Don't writ out compilation databases
"-MJ",
// Continue in the presence of errors during indexing
"-fmodules-validate-once-per-build-session",
.flag("MP", [.singleDash]),
// Don't write out compilation databases
.flag("MJ", [.singleDash]),
// Don't compile
"-c",
]
.flag("c", [.singleDash]),

.flag("fmodules-validate-once-per-build-session", [.singleDash]),

let removeArguments: Set<String> = [
// Disable writing of a depfile
"-MT",
"-MF",
"-MQ",
.option("MT", [.singleDash], [.noSpace, .separatedBySpace]),
.option("MF", [.singleDash], [.noSpace, .separatedBySpace]),
.option("MQ", [.singleDash], [.noSpace, .separatedBySpace]),

// Don't write serialized diagnostic files
"--serialize-diagnostics",
.option("serialize-diagnostics", [.singleDash, .doubleDash], [.separatedBySpace]),

.option("fbuild-session-file", [.singleDash], [.separatedByEqualSign]),
]

var result: [String] = []
result.reserveCapacity(compilerArguments.count)
var iterator = compilerArguments.makeIterator()
while let argument = iterator.next() {
if removeFlags.contains(argument) || argument.starts(with: "-fbuild-session-file=") {
switch optionsToRemove.firstMatch(for: argument) {
case .removeOption:
continue
}
if removeArguments.contains(argument) {
case .removeOptionAndNextArgument:
_ = iterator.next()
continue
case nil:
break
}
result.append(argument)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

@_spi(Testing) import SemanticIndex
import XCTest

final class CompilerCommandLineOptionMatchingTests: XCTestCase {
func testFlags() {
assertOption(.flag("a", [.singleDash]), "-a", .removeOption)
assertOption(.flag("a", [.doubleDash]), "--a", .removeOption)
assertOption(.flag("a", [.singleDash, .doubleDash]), "-a", .removeOption)
assertOption(.flag("a", [.singleDash, .doubleDash]), "--a", .removeOption)
assertOption(.flag("a", [.singleDash]), "-another", nil)
assertOption(.flag("a", [.singleDash]), "--a", nil)
assertOption(.flag("a", [.doubleDash]), "-a", nil)
}

func testOptions() {
assertOption(.option("a", [.singleDash], [.noSpace]), "-a/file.txt", .removeOption)
assertOption(.option("a", [.singleDash], [.noSpace]), "-another", .removeOption)
assertOption(.option("a", [.singleDash], [.separatedByEqualSign]), "-a=/file.txt", .removeOption)
assertOption(.option("a", [.singleDash], [.separatedByEqualSign]), "-a/file.txt", nil)
assertOption(.option("a", [.singleDash], [.separatedBySpace]), "-a", .removeOptionAndNextArgument)
assertOption(.option("a", [.singleDash], [.separatedBySpace]), "-another", nil)
assertOption(.option("a", [.singleDash], [.separatedBySpace]), "-a=/file.txt", nil)
assertOption(.option("a", [.singleDash], [.noSpace, .separatedBySpace]), "-a/file.txt", .removeOption)
assertOption(.option("a", [.singleDash], [.noSpace, .separatedBySpace]), "-a=/file.txt", .removeOption)
assertOption(.option("a", [.singleDash], [.noSpace, .separatedBySpace]), "-a", .removeOptionAndNextArgument)
assertOption(.option("a", [.singleDash], [.separatedByEqualSign, .separatedBySpace]), "-a/file.txt", nil)
assertOption(.option("a", [.singleDash], [.separatedByEqualSign, .separatedBySpace]), "-a=file.txt", .removeOption)
assertOption(
.option("a", [.singleDash], [.separatedByEqualSign, .separatedBySpace]),
"-a",
.removeOptionAndNextArgument
)
}
}

fileprivate func assertOption(
_ option: CompilerCommandLineOption,
_ argument: String,
_ expected: CompilerCommandLineOption.Match?,
file: StaticString = #filePath,
line: UInt = #line
) {
XCTAssertEqual(option.matches(argument: argument), expected, file: file, line: line)
}